diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/BlackScholes.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/BlackScholes.cu new file mode 100644 index 0000000..1512f1a --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/BlackScholes.cu @@ -0,0 +1,265 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This sample evaluates fair call and put prices for a + * given set of European options by Black-Scholes formula. + * See supplied whitepaper for more explanations. + */ + +#include // helper functions for string parsing +#include // helper functions CUDA error checking and initialization + +//////////////////////////////////////////////////////////////////////////////// +// Process an array of optN options on CPU +//////////////////////////////////////////////////////////////////////////////// +extern "C" void BlackScholesCPU(float *h_CallResult, float *h_PutResult, + float *h_StockPrice, float *h_OptionStrike, + float *h_OptionYears, float Riskfree, + float Volatility, int optN); + +//////////////////////////////////////////////////////////////////////////////// +// Process an array of OptN options on GPU +//////////////////////////////////////////////////////////////////////////////// +#include "./include/BlackScholes_kernel.cuh" + +//////////////////////////////////////////////////////////////////////////////// +// Helper function, returning uniformly distributed +// random float in [low, high] range +//////////////////////////////////////////////////////////////////////////////// +float RandFloat(float low, float high) { + float t = (float)rand() / (float)RAND_MAX; + return (1.0f - t) * low + t * high; +} + +//////////////////////////////////////////////////////////////////////////////// +// Data configuration +//////////////////////////////////////////////////////////////////////////////// +const int OPT_N = 4000000; +const int NUM_ITERATIONS = 512; + +const int OPT_SZ = OPT_N * sizeof(float); +const float RISKFREE = 0.02f; +const float VOLATILITY = 0.30f; + +#define DIV_UP(a, b) (((a) + (b)-1) / (b)) + +StopWatchInterface *totalTimer = NULL; + +//////////////////////////////////////////////////////////////////////////////// +// Main program +//////////////////////////////////////////////////////////////////////////////// +int main(int argc, char **argv) { + char* logDir = NULL; + if (checkCmdLineFlag(argc, (const char **)argv, "logDir")) { + getCmdLineArgumentString(argc, (const char **)argv, "logDir=", &logDir); + } + + sdkCreateTimer(&totalTimer); + sdkStartTimer(&totalTimer); + + // Start logs + printf("[%s] - Starting...\n", argv[0]); + + //'h_' prefix - CPU (host) memory space + float + // Results calculated by CPU for reference + *h_CallResultCPU, + *h_PutResultCPU, + // CPU copy of GPU results + *h_CallResultGPU, *h_PutResultGPU, + // CPU instance of input data + *h_StockPrice, *h_OptionStrike, *h_OptionYears; + + //'d_' prefix - GPU (device) memory space + float + // Results calculated by GPU + *d_CallResult, + *d_PutResult, + // GPU instance of input data + *d_StockPrice, *d_OptionStrike, *d_OptionYears; + + double delta, ref, sum_delta, sum_ref, max_delta, L1norm, gpuTime; + + StopWatchInterface *hTimer = NULL; + int i; + + findCudaDevice(argc, (const char **)argv); + + sdkCreateTimer(&hTimer); + + printf("Initializing data...\n"); + printf("...allocating CPU memory for options.\n"); + h_CallResultCPU = (float *)malloc(OPT_SZ); + h_PutResultCPU = (float *)malloc(OPT_SZ); + h_CallResultGPU = (float *)malloc(OPT_SZ); + h_PutResultGPU = (float *)malloc(OPT_SZ); + h_StockPrice = (float *)malloc(OPT_SZ); + h_OptionStrike = (float *)malloc(OPT_SZ); + h_OptionYears = (float *)malloc(OPT_SZ); + + printf("...allocating GPU memory for options.\n"); + checkCudaErrors(cudaMalloc((void **)&d_CallResult, OPT_SZ)); + checkCudaErrors(cudaMalloc((void **)&d_PutResult, OPT_SZ)); + checkCudaErrors(cudaMalloc((void **)&d_StockPrice, OPT_SZ)); + checkCudaErrors(cudaMalloc((void **)&d_OptionStrike, OPT_SZ)); + checkCudaErrors(cudaMalloc((void **)&d_OptionYears, OPT_SZ)); + + printf("...generating input data in CPU mem.\n"); + srand(5347); + + // Generate options set + for (i = 0; i < OPT_N; i++) { + h_CallResultCPU[i] = 0.0f; + h_PutResultCPU[i] = -1.0f; + h_StockPrice[i] = RandFloat(5.0f, 30.0f); + h_OptionStrike[i] = RandFloat(1.0f, 100.0f); + h_OptionYears[i] = RandFloat(0.25f, 10.0f); + } + + printf("...copying input data to GPU mem.\n"); + // Copy options data to GPU memory for further processing + checkCudaErrors( + cudaMemcpy(d_StockPrice, h_StockPrice, OPT_SZ, cudaMemcpyHostToDevice)); + checkCudaErrors(cudaMemcpy(d_OptionStrike, h_OptionStrike, OPT_SZ, + cudaMemcpyHostToDevice)); + checkCudaErrors( + cudaMemcpy(d_OptionYears, h_OptionYears, OPT_SZ, cudaMemcpyHostToDevice)); + printf("Data init done.\n\n"); + + printf("Executing Black-Scholes GPU kernel (%i iterations)...\n", + NUM_ITERATIONS); + checkCudaErrors(cudaDeviceSynchronize()); + sdkResetTimer(&hTimer); + sdkStartTimer(&hTimer); + + for (i = 0; i < NUM_ITERATIONS; i++) { + BlackScholesGPU<<>>( + (float2 *)d_CallResult, (float2 *)d_PutResult, (float2 *)d_StockPrice, + (float2 *)d_OptionStrike, (float2 *)d_OptionYears, RISKFREE, VOLATILITY, + OPT_N); + getLastCudaError("BlackScholesGPU() execution failed\n"); + } + + checkCudaErrors(cudaDeviceSynchronize()); + sdkStopTimer(&hTimer); + gpuTime = sdkGetTimerValue(&hTimer) / NUM_ITERATIONS; + + // Both call and put is calculated + printf("Options count : %i \n", 2 * OPT_N); + printf("BlackScholesGPU() time : %f msec\n", gpuTime); + printf("Effective memory bandwidth: %f GB/s\n", + ((double)(5 * OPT_N * sizeof(float)) * 1E-9) / (gpuTime * 1E-3)); + printf("Gigaoptions per second : %f \n\n", + ((double)(2 * OPT_N) * 1E-9) / (gpuTime * 1E-3)); + + printf( + "BlackScholes, Throughput = %.4f GOptions/s, Time = %.5f s, Size = %u " + "options, NumDevsUsed = %u, Workgroup = %u\n", + (((double)(2.0 * OPT_N) * 1.0E-9) / (gpuTime * 1.0E-3)), gpuTime * 1e-3, + (2 * OPT_N), 1, 128); + + printf("\nReading back GPU results...\n"); + // Read back GPU results to compare them to CPU results + checkCudaErrors(cudaMemcpy(h_CallResultGPU, d_CallResult, OPT_SZ, + cudaMemcpyDeviceToHost)); + checkCudaErrors( + cudaMemcpy(h_PutResultGPU, d_PutResult, OPT_SZ, cudaMemcpyDeviceToHost)); + + printf("Checking the results...\n"); + printf("...running CPU calculations.\n\n"); + // Calculate options values on CPU + BlackScholesCPU(h_CallResultCPU, h_PutResultCPU, h_StockPrice, h_OptionStrike, + h_OptionYears, RISKFREE, VOLATILITY, OPT_N); + + printf("Comparing the results...\n"); + // Calculate max absolute difference and L1 distance + // between CPU and GPU results + sum_delta = 0; + sum_ref = 0; + max_delta = 0; + + for (i = 0; i < OPT_N; i++) { + ref = h_CallResultCPU[i]; + delta = fabs(h_CallResultCPU[i] - h_CallResultGPU[i]); + + if (delta > max_delta) { + max_delta = delta; + } + + sum_delta += delta; + sum_ref += fabs(ref); + } + + L1norm = sum_delta / sum_ref; + printf("L1 norm: %E\n", L1norm); + printf("Max absolute error: %E\n\n", max_delta); + + printf("Shutting down...\n"); + printf("...releasing GPU memory.\n"); + checkCudaErrors(cudaFree(d_OptionYears)); + checkCudaErrors(cudaFree(d_OptionStrike)); + checkCudaErrors(cudaFree(d_StockPrice)); + checkCudaErrors(cudaFree(d_PutResult)); + checkCudaErrors(cudaFree(d_CallResult)); + + printf("...releasing CPU memory.\n"); + free(h_OptionYears); + free(h_OptionStrike); + free(h_StockPrice); + free(h_PutResultGPU); + free(h_CallResultGPU); + free(h_PutResultCPU); + free(h_CallResultCPU); + sdkDeleteTimer(&hTimer); + printf("Shutdown done.\n"); + + printf("\n[BlackScholes] - Test Summary\n"); + + if (L1norm > 1e-6) { + printf("Test failed!\n"); + exit(EXIT_FAILURE); + } + + printf( + "\nNOTE: The CUDA Samples are not meant for performance measurements. " + "Results may vary when GPU Boost is enabled.\n\n"); + printf("Test passed\n"); + + sdkStopTimer(&totalTimer); + double totalTime = sdkGetTimerValue(&totalTimer); + printf("Total time: %f ms \n", totalTime); + if (logDir != NULL) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << totalTime << std::endl; + outFile.close(); + } + + sdkDeleteTimer(&totalTimer); + + exit(EXIT_SUCCESS); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/alignedTypes.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/alignedTypes.cu new file mode 100644 index 0000000..1d91b01 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/alignedTypes.cu @@ -0,0 +1,336 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This is a simple test showing huge access speed gap + * between aligned and misaligned structures + * (those having/missing __align__ keyword). + * It measures per-element copy throughput for + * aligned and misaligned structures on + * big chunks of data. + */ + +// includes, system +#include +#include +#include +#include +#include + +// includes, project +#include // helper functions for CUDA error checking and initialization +#include // helper utility functions + +//////////////////////////////////////////////////////////////////////////////// +// Misaligned types +//////////////////////////////////////////////////////////////////////////////// +typedef unsigned char uint8; + +typedef unsigned short int uint16; + +typedef struct { + unsigned char r, g, b, a; +} RGBA8_misaligned; + +typedef struct { + unsigned int l, a; +} LA32_misaligned; + +typedef struct { + unsigned int r, g, b; +} RGB32_misaligned; + +typedef struct { + unsigned int r, g, b, a; +} RGBA32_misaligned; + +//////////////////////////////////////////////////////////////////////////////// +// Aligned types +//////////////////////////////////////////////////////////////////////////////// +typedef struct __align__(4) { + unsigned char r, g, b, a; +} +RGBA8; + +typedef unsigned int I32; + +typedef struct __align__(8) { + unsigned int l, a; +} +LA32; + +typedef struct __align__(16) { + unsigned int r, g, b; +} +RGB32; + +typedef struct __align__(16) { + unsigned int r, g, b, a; +} +RGBA32; + +//////////////////////////////////////////////////////////////////////////////// +// Because G80 class hardware natively supports global memory operations +// only with data elements of 4, 8 and 16 bytes, if structure size +// exceeds 16 bytes, it can't be efficiently read or written, +// since more than one global memory non-coalescable load/store instructions +// will be generated, even if __align__ option is supplied. +// "Structure of arrays" storage strategy offers best performance +// in general case. See section 5.1.2 of the Programming Guide. +//////////////////////////////////////////////////////////////////////////////// +typedef struct __align__(16) { + RGBA32 c1, c2; +} +RGBA32_2; + +//////////////////////////////////////////////////////////////////////////////// +// Common host and device functions +//////////////////////////////////////////////////////////////////////////////// +// Round a / b to nearest higher integer value +int iDivUp(int a, int b) { return (a % b != 0) ? (a / b + 1) : (a / b); } + +// Round a / b to nearest lower integer value +int iDivDown(int a, int b) { return a / b; } + +// Align a to nearest higher multiple of b +int iAlignUp(int a, int b) { return (a % b != 0) ? (a - a % b + b) : a; } + +// Align a to nearest lower multiple of b +int iAlignDown(int a, int b) { return a - a % b; } + +//////////////////////////////////////////////////////////////////////////////// +// Simple CUDA kernel. +// Copy is carried out on per-element basis, +// so it's not per-byte in case of padded structures. +//////////////////////////////////////////////////////////////////////////////// +template +__global__ void testKernel(TData *d_odata, TData *d_idata, int numElements) { + const int tid = blockDim.x * blockIdx.x + threadIdx.x; + const int numThreads = blockDim.x * gridDim.x; + + for (int pos = tid; pos < numElements; pos += numThreads) { + d_odata[pos] = d_idata[pos]; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Validation routine for simple copy kernel. +// We must know "packed" size of TData (number_of_fields * sizeof(simple_type)) +// and compare only these "packed" parts of the structure, +// containing actual user data. The compiler behavior with padding bytes +// is undefined, since padding is merely a placeholder +// and doesn't contain any user data. +//////////////////////////////////////////////////////////////////////////////// +template +int testCPU(TData *h_odata, TData *h_idata, int numElements, + int packedElementSize) { + for (int pos = 0; pos < numElements; pos++) { + TData src = h_idata[pos]; + TData dst = h_odata[pos]; + + for (int i = 0; i < packedElementSize; i++) + if (((char *)&src)[i] != ((char *)&dst)[i]) { + return 0; + } + } + + return 1; +} + +//////////////////////////////////////////////////////////////////////////////// +// Data configuration +//////////////////////////////////////////////////////////////////////////////// +// Memory chunk size in bytes. Reused for test +const int MEM_SIZE = 50000000; +const int NUM_ITERATIONS = 32; + +// GPU input and output data +unsigned char *d_idata, *d_odata; +// CPU input data and instance of GPU output data +unsigned char *h_idataCPU, *h_odataGPU; +StopWatchInterface *hTimer = NULL; +StopWatchInterface *totalTimer = NULL; + +template +int runTest(int packedElementSize, int memory_size) { + const int totalMemSizeAligned = iAlignDown(memory_size, sizeof(TData)); + const int numElements = iDivDown(memory_size, sizeof(TData)); + + // Clean output buffer before current test + checkCudaErrors(cudaMemset(d_odata, 0, memory_size)); + // Run test + checkCudaErrors(cudaDeviceSynchronize()); + sdkResetTimer(&hTimer); + sdkStartTimer(&hTimer); + + for (int i = 0; i < NUM_ITERATIONS; i++) { + testKernel + <<<64, 256>>>((TData *)d_odata, (TData *)d_idata, numElements); + getLastCudaError("testKernel() execution failed\n"); + } + + checkCudaErrors(cudaDeviceSynchronize()); + sdkStopTimer(&hTimer); + double gpuTime = sdkGetTimerValue(&hTimer) / NUM_ITERATIONS; + printf("Avg. time: %f ms / Copy throughput: %f GB/s.\n", gpuTime, + (double)totalMemSizeAligned / (gpuTime * 0.001 * 1073741824.0)); + + // Read back GPU results and run validation + checkCudaErrors( + cudaMemcpy(h_odataGPU, d_odata, memory_size, cudaMemcpyDeviceToHost)); + int flag = testCPU((TData *)h_odataGPU, (TData *)h_idataCPU, numElements, + packedElementSize); + + printf(flag ? "\tTEST OK\n" : "\tTEST FAILURE\n"); + + return !flag; +} + +int main(int argc, char **argv) { + int i, nTotalFailures = 0; + + char* logDir = NULL; + if (checkCmdLineFlag(argc, (const char **)argv, "logDir")) { + getCmdLineArgumentString(argc, (const char **)argv, "logDir=", &logDir); + } + + sdkCreateTimer(&totalTimer); + sdkStartTimer(&totalTimer); + int devID; + cudaDeviceProp deviceProp; + printf("[%s] - Starting...\n", argv[0]); + + // find first CUDA device + devID = findCudaDevice(argc, (const char **)argv); + + // get number of SMs on this GPU + checkCudaErrors(cudaGetDeviceProperties(&deviceProp, devID)); + printf("[%s] has %d MP(s) x %d (Cores/MP) = %d (Cores)\n", deviceProp.name, + deviceProp.multiProcessorCount, + _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor), + _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) * + deviceProp.multiProcessorCount); + + // Anything that is less than 192 Cores will have a scaled down workload + float scale_factor = + max((192.0f / (_ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) * + (float)deviceProp.multiProcessorCount)), + 1.0f); + + int MemorySize = (int)(MEM_SIZE / scale_factor) & + 0xffffff00; // force multiple of 256 bytes + + printf("> Compute scaling value = %4.2f\n", scale_factor); + printf("> Memory Size = %d\n", MemorySize); + + // sdkCreateTimer(&totalTimer); + // sdkStartTimer(&totalTimer); + sdkCreateTimer(&hTimer); + + printf("Allocating memory...\n"); + h_idataCPU = (unsigned char *)malloc(MemorySize); + h_odataGPU = (unsigned char *)malloc(MemorySize); + checkCudaErrors(cudaMalloc((void **)&d_idata, MemorySize)); + checkCudaErrors(cudaMalloc((void **)&d_odata, MemorySize)); + + printf("Generating host input data array...\n"); + + for (i = 0; i < MemorySize; i++) { + h_idataCPU[i] = (i & 0xFF) + 1; + } + + printf("Uploading input data to GPU memory...\n"); + checkCudaErrors( + cudaMemcpy(d_idata, h_idataCPU, MemorySize, cudaMemcpyHostToDevice)); + + printf("Testing misaligned types...\n"); + printf("uint8...\n"); + nTotalFailures += runTest(1, MemorySize); + + printf("uint16...\n"); + nTotalFailures += runTest(2, MemorySize); + + printf("RGBA8_misaligned...\n"); + nTotalFailures += runTest(4, MemorySize); + + printf("LA32_misaligned...\n"); + nTotalFailures += runTest(8, MemorySize); + + printf("RGB32_misaligned...\n"); + nTotalFailures += runTest(12, MemorySize); + + printf("RGBA32_misaligned...\n"); + nTotalFailures += runTest(16, MemorySize); + + printf("Testing aligned types...\n"); + printf("RGBA8...\n"); + nTotalFailures += runTest(4, MemorySize); + + printf("I32...\n"); + nTotalFailures += runTest(4, MemorySize); + + printf("LA32...\n"); + nTotalFailures += runTest(8, MemorySize); + + printf("RGB32...\n"); + nTotalFailures += runTest(12, MemorySize); + + printf("RGBA32...\n"); + nTotalFailures += runTest(16, MemorySize); + + printf("RGBA32_2...\n"); + nTotalFailures += runTest(32, MemorySize); + + printf("\n[alignedTypes] -> Test Results: %d Failures\n", nTotalFailures); + + printf("Shutting down...\n"); + checkCudaErrors(cudaFree(d_idata)); + checkCudaErrors(cudaFree(d_odata)); + free(h_odataGPU); + free(h_idataCPU); + + sdkStopTimer(&totalTimer); + double totalTime = sdkGetTimerValue(&totalTimer); + printf("Total time: %f ms \n", totalTime); + if (logDir != NULL) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << totalTime << std::endl; + outFile.close(); + } + + sdkDeleteTimer(&hTimer); + sdkDeleteTimer(&totalTimer); + + + if (nTotalFailures != 0) { + printf("Test failed!\n"); + exit(EXIT_FAILURE); + } + + printf("Test passed\n"); + exit(EXIT_SUCCESS); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/asyncAPI.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/asyncAPI.cu new file mode 100644 index 0000000..1af3920 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/asyncAPI.cu @@ -0,0 +1,142 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This sample illustrates the usage of CUDA events for both GPU timing and + * overlapping CPU and GPU execution. Events are inserted into a stream + * of CUDA calls. Since CUDA stream calls are asynchronous, the CPU can + * perform computations while GPU is executing (including DMA memcopies + * between the host and device). CPU can query CUDA events to determine + * whether GPU has completed tasks. + */ + +// includes, system +#include + +// includes CUDA Runtime +#include +// #include + +// includes, project +#include +#include // helper utility functions + +__global__ void increment_kernel(int *g_data, int inc_value) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + g_data[idx] = g_data[idx] + inc_value; +} + +bool correct_output(int *data, const int n, const int x) { + for (int i = 0; i < n; i++) + if (data[i] != x) { + printf("Error! data[%d] = %d, ref = %d\n", i, data[i], x); + return false; + } + + return true; +} + +int main(int argc, char *argv[]) { + int devID; + cudaDeviceProp deviceProps; + + printf("[%s] - Starting...\n", argv[0]); + + // This will pick the best possible CUDA capable device + devID = findCudaDevice(argc, (const char **)argv); + + // get device name + checkCudaErrors(cudaGetDeviceProperties(&deviceProps, devID)); + printf("CUDA device [%s]\n", deviceProps.name); + + int n = 16 * 1024 * 1024; + int nbytes = n * sizeof(int); + int value = 26; + + // allocate host memory + int *a = 0; + checkCudaErrors(cudaMallocHost((void **)&a, nbytes)); + memset(a, 0, nbytes); + + // allocate device memory + int *d_a = 0; + checkCudaErrors(cudaMalloc((void **)&d_a, nbytes)); + checkCudaErrors(cudaMemset(d_a, 255, nbytes)); + + // set kernel launch configuration + dim3 threads = dim3(512, 1); + dim3 blocks = dim3(n / threads.x, 1); + + // create cuda event handles + cudaEvent_t start, stop; + checkCudaErrors(cudaEventCreate(&start)); + checkCudaErrors(cudaEventCreate(&stop)); + + StopWatchInterface *timer = NULL; + sdkCreateTimer(&timer); + sdkResetTimer(&timer); + + checkCudaErrors(cudaDeviceSynchronize()); + float gpu_time = 0.0f; + + // asynchronously issue work to the GPU (all to stream 0) + //checkCudaErrors(cudaProfilerStart()); + sdkStartTimer(&timer); + cudaEventRecord(start, 0); + cudaMemcpyAsync(d_a, a, nbytes, cudaMemcpyHostToDevice, 0); + increment_kernel<<>>(d_a, value); + cudaMemcpyAsync(a, d_a, nbytes, cudaMemcpyDeviceToHost, 0); + cudaEventRecord(stop, 0); + sdkStopTimer(&timer); + //checkCudaErrors(cudaProfilerStop()); + + // have CPU do some work while waiting for stage 1 to finish + unsigned long int counter = 0; + + while (cudaEventQuery(stop) == cudaErrorNotReady) { + counter++; + } + + checkCudaErrors(cudaEventElapsedTime(&gpu_time, start, stop)); + + // print the cpu and gpu times + printf("time spent executing by the GPU: %.2f\n", gpu_time); + printf("time spent by CPU in CUDA calls: %.2f\n", sdkGetTimerValue(&timer)); + printf("CPU executed %lu iterations while waiting for GPU to finish\n", + counter); + + // check the output for correctness + bool bFinalResults = correct_output(a, n, value); + + // release resources + checkCudaErrors(cudaEventDestroy(start)); + checkCudaErrors(cudaEventDestroy(stop)); + checkCudaErrors(cudaFreeHost(a)); + checkCudaErrors(cudaFree(d_a)); + + exit(bFinalResults ? EXIT_SUCCESS : EXIT_FAILURE); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/bandwidthTest.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/bandwidthTest.cu new file mode 100644 index 0000000..ae0fc2b --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/bandwidthTest.cu @@ -0,0 +1,969 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This is a simple test program to measure the memcopy bandwidth of the GPU. + * It can measure device to device copy bandwidth, host to device copy bandwidth + * for pageable and pinned memory, and device to host copy bandwidth for + * pageable and pinned memory. + * + * Usage: + * ./bandwidthTest [option]... + */ + +// CUDA runtime +#include + +// includes +#include // helper functions for CUDA error checking and initialization +#include // helper for shared functions common to CUDA Samples + +#include + +#include +#include +#include + +static const char *sSDKsample = "CUDA Bandwidth Test"; + +// defines, project +#define MEMCOPY_ITERATIONS 100 +#define DEFAULT_SIZE (32 * (1e6)) // 32 M +#define DEFAULT_INCREMENT (4 * (1e6)) // 4 M +#define CACHE_CLEAR_SIZE (16 * (1e6)) // 16 M + +// shmoo mode defines +#define SHMOO_MEMSIZE_MAX (64 * (1e6)) // 64 M +#define SHMOO_MEMSIZE_START (1e3) // 1 KB +#define SHMOO_INCREMENT_1KB (1e3) // 1 KB +#define SHMOO_INCREMENT_2KB (2 * 1e3) // 2 KB +#define SHMOO_INCREMENT_10KB (10 * (1e3)) // 10KB +#define SHMOO_INCREMENT_100KB (100 * (1e3)) // 100 KB +#define SHMOO_INCREMENT_1MB (1e6) // 1 MB +#define SHMOO_INCREMENT_2MB (2 * 1e6) // 2 MB +#define SHMOO_INCREMENT_4MB (4 * 1e6) // 4 MB +#define SHMOO_LIMIT_20KB (20 * (1e3)) // 20 KB +#define SHMOO_LIMIT_50KB (50 * (1e3)) // 50 KB +#define SHMOO_LIMIT_100KB (100 * (1e3)) // 100 KB +#define SHMOO_LIMIT_1MB (1e6) // 1 MB +#define SHMOO_LIMIT_16MB (16 * 1e6) // 16 MB +#define SHMOO_LIMIT_32MB (32 * 1e6) // 32 MB + +// CPU cache flush +#define FLUSH_SIZE (256 * 1024 * 1024) +char *flush_buf; + +// enums, project +enum testMode { QUICK_MODE, RANGE_MODE, SHMOO_MODE }; +enum memcpyKind { DEVICE_TO_HOST, HOST_TO_DEVICE, DEVICE_TO_DEVICE }; +enum printMode { USER_READABLE, CSV }; +enum memoryMode { PINNED, PAGEABLE }; + +const char *sMemoryCopyKind[] = {"Device to Host", "Host to Device", + "Device to Device", NULL}; + +const char *sMemoryMode[] = {"PINNED", "PAGEABLE", NULL}; + +// if true, use CPU based timing for everything +static bool bDontUseGPUTiming; + +int *pArgc = NULL; +char **pArgv = NULL; + +//////////////////////////////////////////////////////////////////////////////// +// declaration, forward +int runTest(const int argc, const char **argv); +void testBandwidth(unsigned int start, unsigned int end, unsigned int increment, + testMode mode, memcpyKind kind, printMode printmode, + memoryMode memMode, int startDevice, int endDevice, bool wc); +void testBandwidthQuick(unsigned int size, memcpyKind kind, printMode printmode, + memoryMode memMode, int startDevice, int endDevice, + bool wc); +void testBandwidthRange(unsigned int start, unsigned int end, + unsigned int increment, memcpyKind kind, + printMode printmode, memoryMode memMode, + int startDevice, int endDevice, bool wc); +void testBandwidthShmoo(memcpyKind kind, printMode printmode, + memoryMode memMode, int startDevice, int endDevice, + bool wc); +float testDeviceToHostTransfer(unsigned int memSize, memoryMode memMode, + bool wc); +float testHostToDeviceTransfer(unsigned int memSize, memoryMode memMode, + bool wc); +float testDeviceToDeviceTransfer(unsigned int memSize); +void printResultsReadable(unsigned int *memSizes, double *bandwidths, + unsigned int count, memcpyKind kind, + memoryMode memMode, int iNumDevs, bool wc); +void printResultsCSV(unsigned int *memSizes, double *bandwidths, + unsigned int count, memcpyKind kind, memoryMode memMode, + int iNumDevs, bool wc); +void printHelp(void); + +//////////////////////////////////////////////////////////////////////////////// +// Program main +//////////////////////////////////////////////////////////////////////////////// +int main(int argc, char **argv) { + pArgc = &argc; + pArgv = argv; + + flush_buf = (char *)malloc(FLUSH_SIZE); + + // set logfile name and start logs + printf("[%s] - Starting...\n", sSDKsample); + + int iRetVal = runTest(argc, (const char **)argv); + + if (iRetVal < 0) { + checkCudaErrors(cudaSetDevice(0)); + } + + // finish + printf("%s\n", (iRetVal == 0) ? "Result = PASS" : "Result = FAIL"); + + printf( + "\nNOTE: The CUDA Samples are not meant for performance measurements. " + "Results may vary when GPU Boost is enabled.\n"); + + free(flush_buf); + + exit((iRetVal == 0) ? EXIT_SUCCESS : EXIT_FAILURE); +} + +/////////////////////////////////////////////////////////////////////////////// +// Parse args, run the appropriate tests +/////////////////////////////////////////////////////////////////////////////// +int runTest(const int argc, const char **argv) { + int start = DEFAULT_SIZE; + int end = DEFAULT_SIZE; + int startDevice = 0; + int endDevice = 0; + int increment = DEFAULT_INCREMENT; + testMode mode = QUICK_MODE; + bool htod = false; + bool dtoh = false; + bool dtod = false; + bool wc = false; + char *modeStr; + char *device = NULL; + printMode printmode = USER_READABLE; + char *memModeStr = NULL; + memoryMode memMode = PINNED; + + // process command line args + if (checkCmdLineFlag(argc, argv, "help")) { + printHelp(); + return 0; + } + + if (checkCmdLineFlag(argc, argv, "csv")) { + printmode = CSV; + } + + if (getCmdLineArgumentString(argc, argv, "memory=", &memModeStr)) { + if (strcmp(memModeStr, "pageable") == 0) { + memMode = PAGEABLE; + } else if (strcmp(memModeStr, "pinned") == 0) { + memMode = PINNED; + } else { + printf("Invalid memory mode - valid modes are pageable or pinned\n"); + printf("See --help for more information\n"); + return -1000; + } + } else { + // default - pinned memory + memMode = PINNED; + } + + if (getCmdLineArgumentString(argc, argv, "device=", &device)) { + int deviceCount; + cudaError_t error_id = cudaGetDeviceCount(&deviceCount); + + if (error_id != cudaSuccess) { + printf("cudaGetDeviceCount returned %d\n-> %s\n", (int)error_id, + cudaGetErrorString(error_id)); + exit(EXIT_FAILURE); + } + + if (deviceCount == 0) { + printf("!!!!!No devices found!!!!!\n"); + return -2000; + } + + if (strcmp(device, "all") == 0) { + printf( + "\n!!!!!Cumulative Bandwidth to be computed from all the devices " + "!!!!!!\n\n"); + startDevice = 0; + endDevice = deviceCount - 1; + } else { + startDevice = endDevice = atoi(device); + + if (startDevice >= deviceCount || startDevice < 0) { + printf( + "\n!!!!!Invalid GPU number %d given hence default gpu %d will be " + "used !!!!!\n", + startDevice, 0); + startDevice = endDevice = 0; + } + } + } + + printf("Running on...\n\n"); + + for (int currentDevice = startDevice; currentDevice <= endDevice; + currentDevice++) { + cudaDeviceProp deviceProp; + cudaError_t error_id = cudaGetDeviceProperties(&deviceProp, currentDevice); + + if (error_id == cudaSuccess) { + printf(" Device %d: %s\n", currentDevice, deviceProp.name); + + if (deviceProp.computeMode == cudaComputeModeProhibited) { + fprintf(stderr, + "Error: device is running in , no " + "threads can use ::cudaSetDevice().\n"); + checkCudaErrors(cudaSetDevice(currentDevice)); + + exit(EXIT_FAILURE); + } + } else { + printf("cudaGetDeviceProperties returned %d\n-> %s\n", (int)error_id, + cudaGetErrorString(error_id)); + checkCudaErrors(cudaSetDevice(currentDevice)); + + exit(EXIT_FAILURE); + } + } + + if (getCmdLineArgumentString(argc, argv, "mode=", &modeStr)) { + // figure out the mode + if (strcmp(modeStr, "quick") == 0) { + printf(" Quick Mode\n\n"); + mode = QUICK_MODE; + } else if (strcmp(modeStr, "shmoo") == 0) { + printf(" Shmoo Mode\n\n"); + mode = SHMOO_MODE; + } else if (strcmp(modeStr, "range") == 0) { + printf(" Range Mode\n\n"); + mode = RANGE_MODE; + } else { + printf("Invalid mode - valid modes are quick, range, or shmoo\n"); + printf("See --help for more information\n"); + return -3000; + } + } else { + // default mode - quick + printf(" Quick Mode\n\n"); + mode = QUICK_MODE; + } + + if (checkCmdLineFlag(argc, argv, "htod")) { + htod = true; + } + + if (checkCmdLineFlag(argc, argv, "dtoh")) { + dtoh = true; + } + + if (checkCmdLineFlag(argc, argv, "dtod")) { + dtod = true; + } + +#if CUDART_VERSION >= 2020 + + if (checkCmdLineFlag(argc, argv, "wc")) { + wc = true; + } + +#endif + + if (checkCmdLineFlag(argc, argv, "cputiming")) { + bDontUseGPUTiming = true; + } + + if (!htod && !dtoh && !dtod) { + // default: All + htod = true; + dtoh = true; + dtod = true; + } + + if (RANGE_MODE == mode) { + if (checkCmdLineFlag(argc, (const char **)argv, "start")) { + start = getCmdLineArgumentInt(argc, argv, "start"); + + if (start <= 0) { + printf("Illegal argument - start must be greater than zero\n"); + return -4000; + } + } else { + printf("Must specify a starting size in range mode\n"); + printf("See --help for more information\n"); + return -5000; + } + + if (checkCmdLineFlag(argc, (const char **)argv, "end")) { + end = getCmdLineArgumentInt(argc, argv, "end"); + + if (end <= 0) { + printf("Illegal argument - end must be greater than zero\n"); + return -6000; + } + + if (start > end) { + printf("Illegal argument - start is greater than end\n"); + return -7000; + } + } else { + printf("Must specify an end size in range mode.\n"); + printf("See --help for more information\n"); + return -8000; + } + + if (checkCmdLineFlag(argc, argv, "increment")) { + increment = getCmdLineArgumentInt(argc, argv, "increment"); + + if (increment <= 0) { + printf("Illegal argument - increment must be greater than zero\n"); + return -9000; + } + } else { + printf("Must specify an increment in user mode\n"); + printf("See --help for more information\n"); + return -10000; + } + } + + if (htod) { + testBandwidth((unsigned int)start, (unsigned int)end, + (unsigned int)increment, mode, HOST_TO_DEVICE, printmode, + memMode, startDevice, endDevice, wc); + } + + if (dtoh) { + testBandwidth((unsigned int)start, (unsigned int)end, + (unsigned int)increment, mode, DEVICE_TO_HOST, printmode, + memMode, startDevice, endDevice, wc); + } + + if (dtod) { + testBandwidth((unsigned int)start, (unsigned int)end, + (unsigned int)increment, mode, DEVICE_TO_DEVICE, printmode, + memMode, startDevice, endDevice, wc); + } + + // Ensure that we reset all CUDA Devices in question + for (int nDevice = startDevice; nDevice <= endDevice; nDevice++) { + cudaSetDevice(nDevice); + } + + return 0; +} + +/////////////////////////////////////////////////////////////////////////////// +// Run a bandwidth test +/////////////////////////////////////////////////////////////////////////////// +void testBandwidth(unsigned int start, unsigned int end, unsigned int increment, + testMode mode, memcpyKind kind, printMode printmode, + memoryMode memMode, int startDevice, int endDevice, + bool wc) { + switch (mode) { + case QUICK_MODE: + testBandwidthQuick(DEFAULT_SIZE, kind, printmode, memMode, startDevice, + endDevice, wc); + break; + + case RANGE_MODE: + testBandwidthRange(start, end, increment, kind, printmode, memMode, + startDevice, endDevice, wc); + break; + + case SHMOO_MODE: + testBandwidthShmoo(kind, printmode, memMode, startDevice, endDevice, wc); + break; + + default: + break; + } +} + +////////////////////////////////////////////////////////////////////// +// Run a quick mode bandwidth test +////////////////////////////////////////////////////////////////////// +void testBandwidthQuick(unsigned int size, memcpyKind kind, printMode printmode, + memoryMode memMode, int startDevice, int endDevice, + bool wc) { + testBandwidthRange(size, size, DEFAULT_INCREMENT, kind, printmode, memMode, + startDevice, endDevice, wc); +} + +/////////////////////////////////////////////////////////////////////// +// Run a range mode bandwidth test +////////////////////////////////////////////////////////////////////// +void testBandwidthRange(unsigned int start, unsigned int end, + unsigned int increment, memcpyKind kind, + printMode printmode, memoryMode memMode, + int startDevice, int endDevice, bool wc) { + // count the number of copies we're going to run + unsigned int count = 1 + ((end - start) / increment); + + unsigned int *memSizes = (unsigned int *)malloc(count * sizeof(unsigned int)); + double *bandwidths = (double *)malloc(count * sizeof(double)); + + // Before calculating the cumulative bandwidth, initialize bandwidths array to + // NULL + for (unsigned int i = 0; i < count; i++) { + bandwidths[i] = 0.0; + } + + // Use the device asked by the user + for (int currentDevice = startDevice; currentDevice <= endDevice; + currentDevice++) { + cudaSetDevice(currentDevice); + + // run each of the copies + for (unsigned int i = 0; i < count; i++) { + memSizes[i] = start + i * increment; + + switch (kind) { + case DEVICE_TO_HOST: + bandwidths[i] += testDeviceToHostTransfer(memSizes[i], memMode, wc); + break; + + case HOST_TO_DEVICE: + bandwidths[i] += testHostToDeviceTransfer(memSizes[i], memMode, wc); + break; + + case DEVICE_TO_DEVICE: + bandwidths[i] += testDeviceToDeviceTransfer(memSizes[i]); + break; + } + } + } // Complete the bandwidth computation on all the devices + + // print results + if (printmode == CSV) { + printResultsCSV(memSizes, bandwidths, count, kind, memMode, + (1 + endDevice - startDevice), wc); + } else { + printResultsReadable(memSizes, bandwidths, count, kind, memMode, + (1 + endDevice - startDevice), wc); + } + + // clean up + free(memSizes); + free(bandwidths); +} + +////////////////////////////////////////////////////////////////////////////// +// Intense shmoo mode - covers a large range of values with varying increments +////////////////////////////////////////////////////////////////////////////// +void testBandwidthShmoo(memcpyKind kind, printMode printmode, + memoryMode memMode, int startDevice, int endDevice, + bool wc) { + // count the number of copies to make + unsigned int count = + 1 + (SHMOO_LIMIT_20KB / SHMOO_INCREMENT_1KB) + + ((SHMOO_LIMIT_50KB - SHMOO_LIMIT_20KB) / SHMOO_INCREMENT_2KB) + + ((SHMOO_LIMIT_100KB - SHMOO_LIMIT_50KB) / SHMOO_INCREMENT_10KB) + + ((SHMOO_LIMIT_1MB - SHMOO_LIMIT_100KB) / SHMOO_INCREMENT_100KB) + + ((SHMOO_LIMIT_16MB - SHMOO_LIMIT_1MB) / SHMOO_INCREMENT_1MB) + + ((SHMOO_LIMIT_32MB - SHMOO_LIMIT_16MB) / SHMOO_INCREMENT_2MB) + + ((SHMOO_MEMSIZE_MAX - SHMOO_LIMIT_32MB) / SHMOO_INCREMENT_4MB); + + unsigned int *memSizes = (unsigned int *)malloc(count * sizeof(unsigned int)); + double *bandwidths = (double *)malloc(count * sizeof(double)); + + // Before calculating the cumulative bandwidth, initialize bandwidths array to + // NULL + for (unsigned int i = 0; i < count; i++) { + bandwidths[i] = 0.0; + } + + // Use the device asked by the user + for (int currentDevice = startDevice; currentDevice <= endDevice; + currentDevice++) { + cudaSetDevice(currentDevice); + // Run the shmoo + int iteration = 0; + unsigned int memSize = 0; + + while (memSize <= SHMOO_MEMSIZE_MAX) { + if (memSize < SHMOO_LIMIT_20KB) { + memSize += SHMOO_INCREMENT_1KB; + } else if (memSize < SHMOO_LIMIT_50KB) { + memSize += SHMOO_INCREMENT_2KB; + } else if (memSize < SHMOO_LIMIT_100KB) { + memSize += SHMOO_INCREMENT_10KB; + } else if (memSize < SHMOO_LIMIT_1MB) { + memSize += SHMOO_INCREMENT_100KB; + } else if (memSize < SHMOO_LIMIT_16MB) { + memSize += SHMOO_INCREMENT_1MB; + } else if (memSize < SHMOO_LIMIT_32MB) { + memSize += SHMOO_INCREMENT_2MB; + } else { + memSize += SHMOO_INCREMENT_4MB; + } + + memSizes[iteration] = memSize; + + switch (kind) { + case DEVICE_TO_HOST: + bandwidths[iteration] += + testDeviceToHostTransfer(memSizes[iteration], memMode, wc); + break; + + case HOST_TO_DEVICE: + bandwidths[iteration] += + testHostToDeviceTransfer(memSizes[iteration], memMode, wc); + break; + + case DEVICE_TO_DEVICE: + bandwidths[iteration] += + testDeviceToDeviceTransfer(memSizes[iteration]); + break; + } + + iteration++; + printf("."); + fflush(0); + } + } // Complete the bandwidth computation on all the devices + + // print results + printf("\n"); + + if (CSV == printmode) { + printResultsCSV(memSizes, bandwidths, count, kind, memMode, + (1 + endDevice - startDevice), wc); + } else { + printResultsReadable(memSizes, bandwidths, count, kind, memMode, + (1 + endDevice - startDevice), wc); + } + + // clean up + free(memSizes); + free(bandwidths); +} + +/////////////////////////////////////////////////////////////////////////////// +// test the bandwidth of a device to host memcopy of a specific size +/////////////////////////////////////////////////////////////////////////////// +float testDeviceToHostTransfer(unsigned int memSize, memoryMode memMode, + bool wc) { + StopWatchInterface *timer = NULL; + float elapsedTimeInMs = 0.0f; + float bandwidthInGBs = 0.0f; + unsigned char *h_idata = NULL; + unsigned char *h_odata = NULL; + cudaEvent_t start, stop; + + sdkCreateTimer(&timer); + checkCudaErrors(cudaEventCreate(&start)); + checkCudaErrors(cudaEventCreate(&stop)); + + // allocate host memory + if (PINNED == memMode) { + // pinned memory mode - use special function to get OS-pinned memory +#if CUDART_VERSION >= 2020 + checkCudaErrors(cudaHostAlloc((void **)&h_idata, memSize, + (wc) ? cudaHostAllocWriteCombined : 0)); + checkCudaErrors(cudaHostAlloc((void **)&h_odata, memSize, + (wc) ? cudaHostAllocWriteCombined : 0)); +#else + checkCudaErrors(cudaMallocHost((void **)&h_idata, memSize)); + checkCudaErrors(cudaMallocHost((void **)&h_odata, memSize)); +#endif + } else { + // pageable memory mode - use malloc + h_idata = (unsigned char *)malloc(memSize); + h_odata = (unsigned char *)malloc(memSize); + + if (h_idata == 0 || h_odata == 0) { + fprintf(stderr, "Not enough memory avaialable on host to run test!\n"); + exit(EXIT_FAILURE); + } + } + + // initialize the memory + for (unsigned int i = 0; i < memSize / sizeof(unsigned char); i++) { + h_idata[i] = (unsigned char)(i & 0xff); + } + + // allocate device memory + unsigned char *d_idata; + checkCudaErrors(cudaMalloc((void **)&d_idata, memSize)); + + // initialize the device memory + checkCudaErrors( + cudaMemcpy(d_idata, h_idata, memSize, cudaMemcpyHostToDevice)); + + // copy data from GPU to Host + if (PINNED == memMode) { + if (bDontUseGPUTiming) sdkStartTimer(&timer); + checkCudaErrors(cudaEventRecord(start, 0)); + for (unsigned int i = 0; i < MEMCOPY_ITERATIONS; i++) { + checkCudaErrors(cudaMemcpyAsync(h_odata, d_idata, memSize, + cudaMemcpyDeviceToHost, 0)); + } + checkCudaErrors(cudaEventRecord(stop, 0)); + checkCudaErrors(cudaDeviceSynchronize()); + checkCudaErrors(cudaEventElapsedTime(&elapsedTimeInMs, start, stop)); + if (bDontUseGPUTiming) { + sdkStopTimer(&timer); + elapsedTimeInMs = sdkGetTimerValue(&timer); + sdkResetTimer(&timer); + } + } else { + elapsedTimeInMs = 0; + for (unsigned int i = 0; i < MEMCOPY_ITERATIONS; i++) { + sdkStartTimer(&timer); + checkCudaErrors( + cudaMemcpy(h_odata, d_idata, memSize, cudaMemcpyDeviceToHost)); + sdkStopTimer(&timer); + elapsedTimeInMs += sdkGetTimerValue(&timer); + sdkResetTimer(&timer); + memset(flush_buf, i, FLUSH_SIZE); + } + } + + // calculate bandwidth in GB/s + double time_s = elapsedTimeInMs / 1e3; + bandwidthInGBs = (memSize * (float)MEMCOPY_ITERATIONS) / (double)1e9; + bandwidthInGBs = bandwidthInGBs / time_s; + // clean up memory + checkCudaErrors(cudaEventDestroy(stop)); + checkCudaErrors(cudaEventDestroy(start)); + sdkDeleteTimer(&timer); + + if (PINNED == memMode) { + checkCudaErrors(cudaFreeHost(h_idata)); + checkCudaErrors(cudaFreeHost(h_odata)); + } else { + free(h_idata); + free(h_odata); + } + + checkCudaErrors(cudaFree(d_idata)); + + return bandwidthInGBs; +} + +/////////////////////////////////////////////////////////////////////////////// +//! test the bandwidth of a host to device memcopy of a specific size +/////////////////////////////////////////////////////////////////////////////// +float testHostToDeviceTransfer(unsigned int memSize, memoryMode memMode, + bool wc) { + StopWatchInterface *timer = NULL; + float elapsedTimeInMs = 0.0f; + float bandwidthInGBs = 0.0f; + cudaEvent_t start, stop; + sdkCreateTimer(&timer); + checkCudaErrors(cudaEventCreate(&start)); + checkCudaErrors(cudaEventCreate(&stop)); + + // allocate host memory + unsigned char *h_odata = NULL; + + if (PINNED == memMode) { +#if CUDART_VERSION >= 2020 + // pinned memory mode - use special function to get OS-pinned memory + checkCudaErrors(cudaHostAlloc((void **)&h_odata, memSize, + (wc) ? cudaHostAllocWriteCombined : 0)); +#else + // pinned memory mode - use special function to get OS-pinned memory + checkCudaErrors(cudaMallocHost((void **)&h_odata, memSize)); +#endif + } else { + // pageable memory mode - use malloc + h_odata = (unsigned char *)malloc(memSize); + + if (h_odata == 0) { + fprintf(stderr, "Not enough memory available on host to run test!\n"); + exit(EXIT_FAILURE); + } + } + + unsigned char *h_cacheClear1 = (unsigned char *)malloc(CACHE_CLEAR_SIZE); + unsigned char *h_cacheClear2 = (unsigned char *)malloc(CACHE_CLEAR_SIZE); + + if (h_cacheClear1 == 0 || h_cacheClear2 == 0) { + fprintf(stderr, "Not enough memory available on host to run test!\n"); + exit(EXIT_FAILURE); + } + + // initialize the memory + for (unsigned int i = 0; i < memSize / sizeof(unsigned char); i++) { + h_odata[i] = (unsigned char)(i & 0xff); + } + + for (unsigned int i = 0; i < CACHE_CLEAR_SIZE / sizeof(unsigned char); i++) { + h_cacheClear1[i] = (unsigned char)(i & 0xff); + h_cacheClear2[i] = (unsigned char)(0xff - (i & 0xff)); + } + + // allocate device memory + unsigned char *d_idata; + checkCudaErrors(cudaMalloc((void **)&d_idata, memSize)); + + // copy host memory to device memory + if (PINNED == memMode) { + if (bDontUseGPUTiming) sdkStartTimer(&timer); + checkCudaErrors(cudaEventRecord(start, 0)); + for (unsigned int i = 0; i < MEMCOPY_ITERATIONS; i++) { + checkCudaErrors(cudaMemcpyAsync(d_idata, h_odata, memSize, + cudaMemcpyHostToDevice, 0)); + } + checkCudaErrors(cudaEventRecord(stop, 0)); + checkCudaErrors(cudaDeviceSynchronize()); + checkCudaErrors(cudaEventElapsedTime(&elapsedTimeInMs, start, stop)); + if (bDontUseGPUTiming) { + sdkStopTimer(&timer); + elapsedTimeInMs = sdkGetTimerValue(&timer); + sdkResetTimer(&timer); + } + } else { + elapsedTimeInMs = 0; + for (unsigned int i = 0; i < MEMCOPY_ITERATIONS; i++) { + sdkStartTimer(&timer); + checkCudaErrors( + cudaMemcpy(d_idata, h_odata, memSize, cudaMemcpyHostToDevice)); + sdkStopTimer(&timer); + elapsedTimeInMs += sdkGetTimerValue(&timer); + sdkResetTimer(&timer); + memset(flush_buf, i, FLUSH_SIZE); + } + } + + // calculate bandwidth in GB/s + double time_s = elapsedTimeInMs / 1e3; + bandwidthInGBs = (memSize * (float)MEMCOPY_ITERATIONS) / (double)1e9; + bandwidthInGBs = bandwidthInGBs / time_s; + // clean up memory + checkCudaErrors(cudaEventDestroy(stop)); + checkCudaErrors(cudaEventDestroy(start)); + sdkDeleteTimer(&timer); + + if (PINNED == memMode) { + checkCudaErrors(cudaFreeHost(h_odata)); + } else { + free(h_odata); + } + + free(h_cacheClear1); + free(h_cacheClear2); + checkCudaErrors(cudaFree(d_idata)); + + return bandwidthInGBs; +} + +/////////////////////////////////////////////////////////////////////////////// +//! test the bandwidth of a device to device memcopy of a specific size +/////////////////////////////////////////////////////////////////////////////// +float testDeviceToDeviceTransfer(unsigned int memSize) { + StopWatchInterface *timer = NULL; + float elapsedTimeInMs = 0.0f; + float bandwidthInGBs = 0.0f; + cudaEvent_t start, stop; + + sdkCreateTimer(&timer); + checkCudaErrors(cudaEventCreate(&start)); + checkCudaErrors(cudaEventCreate(&stop)); + + // allocate host memory + unsigned char *h_idata = (unsigned char *)malloc(memSize); + + if (h_idata == 0) { + fprintf(stderr, "Not enough memory avaialable on host to run test!\n"); + exit(EXIT_FAILURE); + } + + // initialize the host memory + for (unsigned int i = 0; i < memSize / sizeof(unsigned char); i++) { + h_idata[i] = (unsigned char)(i & 0xff); + } + + // allocate device memory + unsigned char *d_idata; + checkCudaErrors(cudaMalloc((void **)&d_idata, memSize)); + unsigned char *d_odata; + checkCudaErrors(cudaMalloc((void **)&d_odata, memSize)); + + // initialize memory + checkCudaErrors( + cudaMemcpy(d_idata, h_idata, memSize, cudaMemcpyHostToDevice)); + + // run the memcopy + sdkStartTimer(&timer); + checkCudaErrors(cudaEventRecord(start, 0)); + + for (unsigned int i = 0; i < MEMCOPY_ITERATIONS; i++) { + checkCudaErrors( + cudaMemcpy(d_odata, d_idata, memSize, cudaMemcpyDeviceToDevice)); + } + + checkCudaErrors(cudaEventRecord(stop, 0)); + + // Since device to device memory copies are non-blocking, + // cudaDeviceSynchronize() is required in order to get + // proper timing. + checkCudaErrors(cudaDeviceSynchronize()); + + // get the total elapsed time in ms + sdkStopTimer(&timer); + checkCudaErrors(cudaEventElapsedTime(&elapsedTimeInMs, start, stop)); + + if (bDontUseGPUTiming) { + elapsedTimeInMs = sdkGetTimerValue(&timer); + } + + // calculate bandwidth in GB/s + double time_s = elapsedTimeInMs / 1e3; + bandwidthInGBs = (2.0f * memSize * (float)MEMCOPY_ITERATIONS) / (double)1e9; + bandwidthInGBs = bandwidthInGBs / time_s; + + // clean up memory + sdkDeleteTimer(&timer); + free(h_idata); + checkCudaErrors(cudaEventDestroy(stop)); + checkCudaErrors(cudaEventDestroy(start)); + checkCudaErrors(cudaFree(d_idata)); + checkCudaErrors(cudaFree(d_odata)); + + return bandwidthInGBs; +} + +///////////////////////////////////////////////////////// +// print results in an easily read format +//////////////////////////////////////////////////////// +void printResultsReadable(unsigned int *memSizes, double *bandwidths, + unsigned int count, memcpyKind kind, + memoryMode memMode, int iNumDevs, bool wc) { + printf(" %s Bandwidth, %i Device(s)\n", sMemoryCopyKind[kind], iNumDevs); + printf(" %s Memory Transfers\n", sMemoryMode[memMode]); + + if (wc) { + printf(" Write-Combined Memory Writes are Enabled"); + } + + printf(" Transfer Size (Bytes)\tBandwidth(GB/s)\n"); + unsigned int i; + + for (i = 0; i < (count - 1); i++) { + printf(" %u\t\t\t%s%.1f\n", memSizes[i], + (memSizes[i] < 10000) ? "\t" : "", bandwidths[i]); + } + + printf(" %u\t\t\t%s%.1f\n\n", memSizes[i], + (memSizes[i] < 10000) ? "\t" : "", bandwidths[i]); +} + +/////////////////////////////////////////////////////////////////////////// +// print results in a database format +/////////////////////////////////////////////////////////////////////////// +void printResultsCSV(unsigned int *memSizes, double *bandwidths, + unsigned int count, memcpyKind kind, memoryMode memMode, + int iNumDevs, bool wc) { + std::string sConfig; + + // log config information + if (kind == DEVICE_TO_DEVICE) { + sConfig += "D2D"; + } else { + if (kind == DEVICE_TO_HOST) { + sConfig += "D2H"; + } else if (kind == HOST_TO_DEVICE) { + sConfig += "H2D"; + } + + if (memMode == PAGEABLE) { + sConfig += "-Paged"; + } else if (memMode == PINNED) { + sConfig += "-Pinned"; + + if (wc) { + sConfig += "-WriteCombined"; + } + } + } + + unsigned int i; + double dSeconds = 0.0; + + for (i = 0; i < count; i++) { + dSeconds = (double)memSizes[i] / (bandwidths[i] * (double)(1e9)); + printf( + "bandwidthTest-%s, Bandwidth = %.1f GB/s, Time = %.5f s, Size = %u " + "bytes, NumDevsUsed = %d\n", + sConfig.c_str(), bandwidths[i], dSeconds, memSizes[i], iNumDevs); + } +} + +/////////////////////////////////////////////////////////////////////////// +// Print help screen +/////////////////////////////////////////////////////////////////////////// +void printHelp(void) { + printf("Usage: bandwidthTest [OPTION]...\n"); + printf( + "Test the bandwidth for device to host, host to device, and device to " + "device transfers\n"); + printf("\n"); + printf( + "Example: measure the bandwidth of device to host pinned memory copies " + "in the range 1024 Bytes to 102400 Bytes in 1024 Byte increments\n"); + printf( + "./bandwidthTest --memory=pinned --mode=range --start=1024 --end=102400 " + "--increment=1024 --dtoh\n"); + + printf("\n"); + printf("Options:\n"); + printf("--help\tDisplay this help menu\n"); + printf("--csv\tPrint results as a CSV\n"); + printf("--device=[deviceno]\tSpecify the device device to be used\n"); + printf(" all - compute cumulative bandwidth on all the devices\n"); + printf(" 0,1,2,...,n - Specify any particular device to be used\n"); + printf("--memory=[MEMMODE]\tSpecify which memory mode to use\n"); + printf(" pageable - pageable memory\n"); + printf(" pinned - non-pageable system memory\n"); + printf("--mode=[MODE]\tSpecify the mode to use\n"); + printf(" quick - performs a quick measurement\n"); + printf(" range - measures a user-specified range of values\n"); + printf(" shmoo - performs an intense shmoo of a large range of values\n"); + + printf("--htod\tMeasure host to device transfers\n"); + printf("--dtoh\tMeasure device to host transfers\n"); + printf("--dtod\tMeasure device to device transfers\n"); +#if CUDART_VERSION >= 2020 + printf("--wc\tAllocate pinned memory as write-combined\n"); +#endif + printf("--cputiming\tForce CPU-based timing always\n"); + + printf("Range mode options\n"); + printf("--start=[SIZE]\tStarting transfer size in bytes\n"); + printf("--end=[SIZE]\tEnding transfer size in bytes\n"); + printf("--increment=[SIZE]\tIncrement size in bytes\n"); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/clock.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/clock.cu new file mode 100644 index 0000000..bb97e1b --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/clock.cu @@ -0,0 +1,178 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This example shows how to use the clock function to measure the performance + * of block of threads of a kernel accurately. Blocks are executed in parallel + * and out of order. Since there's no synchronization mechanism between blocks, + * we measure the clock once for each block. The clock samples are written to + * device memory. + */ + +// System includes +#include +#include +#include +#include + +// CUDA runtime +#include + +// helper functions and utilities to work with CUDA +#include +#include + +// This kernel computes a standard parallel reduction and evaluates the +// time it takes to do that for each block. The timing results are stored +// in device memory. +__global__ static void timedReduction(const float *input, float *output, + clock_t *timer) { + // __shared__ float shared[2 * blockDim.x]; + extern __shared__ float shared[]; + + const int tid = threadIdx.x; + const int bid = blockIdx.x; + + if (tid == 0) timer[bid] = clock(); + + // Copy input. + shared[tid] = input[tid]; + shared[tid + blockDim.x] = input[tid + blockDim.x]; + + // Perform reduction to find minimum. + for (int d = blockDim.x; d > 0; d /= 2) { + __syncthreads(); + + if (tid < d) { + float f0 = shared[tid]; + float f1 = shared[tid + d]; + + if (f1 < f0) { + shared[tid] = f1; + } + } + } + + // Write result. + if (tid == 0) output[bid] = shared[0]; + + __syncthreads(); + + if (tid == 0) timer[bid + gridDim.x] = clock(); +} + +#define NUM_BLOCKS 64 +#define NUM_THREADS 256 + +// It's interesting to change the number of blocks and the number of threads to +// understand how to keep the hardware busy. +// +// Here are some numbers I get on my G80: +// blocks - clocks +// 1 - 3096 +// 8 - 3232 +// 16 - 3364 +// 32 - 4615 +// 64 - 9981 +// +// With less than 16 blocks some of the multiprocessors of the device are idle. +// With more than 16 you are using all the multiprocessors, but there's only one +// block per multiprocessor and that doesn't allow you to hide the latency of +// the memory. With more than 32 the speed scales linearly. + +// Start the main CUDA Sample here + +StopWatchInterface *totalTimer = NULL; + +int main(int argc, char **argv) { + + char* logDir = NULL; + if (checkCmdLineFlag(argc, (const char **)argv, "logDir")) { + getCmdLineArgumentString(argc, (const char **)argv, "logDir=", &logDir); + } + + sdkCreateTimer(&totalTimer); + sdkStartTimer(&totalTimer); + + printf("CUDA Clock sample\n"); + + // This will pick the best possible CUDA capable device + int dev = findCudaDevice(argc, (const char **)argv); + + float *dinput = NULL; + float *doutput = NULL; + clock_t *dtimer = NULL; + + clock_t timer[NUM_BLOCKS * 2]; + float input[NUM_THREADS * 2]; + + for (int i = 0; i < NUM_THREADS * 2; i++) { + input[i] = (float)i; + } + + checkCudaErrors( + cudaMalloc((void **)&dinput, sizeof(float) * NUM_THREADS * 2)); + checkCudaErrors(cudaMalloc((void **)&doutput, sizeof(float) * NUM_BLOCKS)); + checkCudaErrors( + cudaMalloc((void **)&dtimer, sizeof(clock_t) * NUM_BLOCKS * 2)); + + checkCudaErrors(cudaMemcpy(dinput, input, sizeof(float) * NUM_THREADS * 2, + cudaMemcpyHostToDevice)); + + timedReduction<<>>( + dinput, doutput, dtimer); + + checkCudaErrors(cudaMemcpy(timer, dtimer, sizeof(clock_t) * NUM_BLOCKS * 2, + cudaMemcpyDeviceToHost)); + + checkCudaErrors(cudaFree(dinput)); + checkCudaErrors(cudaFree(doutput)); + checkCudaErrors(cudaFree(dtimer)); + + long double avgElapsedClocks = 0; + + for (int i = 0; i < NUM_BLOCKS; i++) { + avgElapsedClocks += (long double)(timer[i + NUM_BLOCKS] - timer[i]); + } + + avgElapsedClocks = avgElapsedClocks / NUM_BLOCKS; + printf("Average clocks/block = %Lf\n", avgElapsedClocks); + + sdkStopTimer(&totalTimer); + double totalTime = sdkGetTimerValue(&totalTimer); + printf("Total time: %f ms \n", totalTime); + if (logDir != NULL) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << totalTime << std::endl; + outFile.close(); + } + + sdkDeleteTimer(&totalTimer); + + + return EXIT_SUCCESS; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/convolutionSeparable.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/convolutionSeparable.cu new file mode 100644 index 0000000..1accf1a --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/convolutionSeparable.cu @@ -0,0 +1,182 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* +* This sample implements a separable convolution filter +* of a 2D image with an arbitrary kernel. +*/ + + +// Utilities and system includes +#include +#include +#include + +#include "./include/convolutionSeparable_common.cuh" +StopWatchInterface *totalTimer = NULL; +//////////////////////////////////////////////////////////////////////////////// +// Main program +//////////////////////////////////////////////////////////////////////////////// +int main(int argc, char **argv) { + cudaDeviceSynchronize(); + char* logDir = NULL; + if (checkCmdLineFlag(argc, (const char **)argv, "logDir")) { + getCmdLineArgumentString(argc, (const char **)argv, "logDir=", &logDir); + } + + sdkCreateTimer(&totalTimer); + sdkStartTimer(&totalTimer); + // start logs + printf("[%s] - Starting...\n", argv[0]); + + float *h_Kernel, *h_Input, *h_Buffer, *h_OutputCPU, *h_OutputGPU; + + float *d_Input, *d_Output, *d_Buffer; + + const int imageW = 3072; + const int imageH = 3072; + const int iterations = 16; + + StopWatchInterface *hTimer = NULL; + + // Use command-line specified CUDA device, otherwise use device with highest + // Gflops/s + findCudaDevice(argc, (const char **)argv); + + sdkCreateTimer(&hTimer); + + printf("Image Width x Height = %i x %i\n\n", imageW, imageH); + printf("Allocating and initializing host arrays...\n"); + h_Kernel = (float *)malloc(KERNEL_LENGTH * sizeof(float)); + h_Input = (float *)malloc(imageW * imageH * sizeof(float)); + h_Buffer = (float *)malloc(imageW * imageH * sizeof(float)); + h_OutputCPU = (float *)malloc(imageW * imageH * sizeof(float)); + h_OutputGPU = (float *)malloc(imageW * imageH * sizeof(float)); + srand(200); + + for (unsigned int i = 0; i < KERNEL_LENGTH; i++) { + h_Kernel[i] = (float)(rand() % 16); + } + + for (unsigned i = 0; i < imageW * imageH; i++) { + h_Input[i] = (float)(rand() % 16); + } + + printf("Allocating and initializing CUDA arrays...\n"); + checkCudaErrors( + cudaMalloc((void **)&d_Input, imageW * imageH * sizeof(float))); + checkCudaErrors( + cudaMalloc((void **)&d_Output, imageW * imageH * sizeof(float))); + checkCudaErrors( + cudaMalloc((void **)&d_Buffer, imageW * imageH * sizeof(float))); + + setConvolutionKernel(h_Kernel); + checkCudaErrors(cudaMemcpy(d_Input, h_Input, imageW * imageH * sizeof(float), + cudaMemcpyHostToDevice)); + + printf("Running GPU convolution (%u identical iterations)...\n\n", + iterations); + + for (int i = -1; i < iterations; i++) { + // i == -1 -- warmup iteration + if (i == 0) { + checkCudaErrors(cudaDeviceSynchronize()); + sdkResetTimer(&hTimer); + sdkStartTimer(&hTimer); + } + + convolutionRowsGPU(d_Buffer, d_Input, imageW, imageH); + + convolutionColumnsGPU(d_Output, d_Buffer, imageW, imageH); + } + + checkCudaErrors(cudaDeviceSynchronize()); + sdkStopTimer(&hTimer); + double gpuTime = 0.001 * sdkGetTimerValue(&hTimer) / (double)iterations; + printf( + "convolutionSeparable, Throughput = %.4f MPixels/sec, Time = %.5f s, " + "Size = %u Pixels, NumDevsUsed = %i, Workgroup = %u\n", + (1.0e-6 * (double)(imageW * imageH) / gpuTime), gpuTime, + (imageW * imageH), 1, 0); + + printf("\nReading back GPU results...\n\n"); + checkCudaErrors(cudaMemcpy(h_OutputGPU, d_Output, + imageW * imageH * sizeof(float), + cudaMemcpyDeviceToHost)); + + printf("Checking the results...\n"); + printf(" ...running convolutionRowCPU()\n"); + convolutionRowCPU(h_Buffer, h_Input, h_Kernel, imageW, imageH, KERNEL_RADIUS); + + printf(" ...running convolutionColumnCPU()\n"); + convolutionColumnCPU(h_OutputCPU, h_Buffer, h_Kernel, imageW, imageH, + KERNEL_RADIUS); + + printf(" ...comparing the results\n"); + double sum = 0, delta = 0; + + for (unsigned i = 0; i < imageW * imageH; i++) { + delta += + (h_OutputGPU[i] - h_OutputCPU[i]) * (h_OutputGPU[i] - h_OutputCPU[i]); + sum += h_OutputCPU[i] * h_OutputCPU[i]; + } + + double L2norm = sqrt(delta / sum); + printf(" ...Relative L2 norm: %E\n\n", L2norm); + printf("Shutting down...\n"); + + checkCudaErrors(cudaFree(d_Buffer)); + checkCudaErrors(cudaFree(d_Output)); + checkCudaErrors(cudaFree(d_Input)); + free(h_OutputGPU); + free(h_OutputCPU); + free(h_Buffer); + free(h_Input); + free(h_Kernel); + + sdkDeleteTimer(&hTimer); + + if (L2norm > 1e-6) { + printf("Test failed!\n"); + exit(EXIT_FAILURE); + } + + printf("Test passed\n"); + + sdkStopTimer(&totalTimer); + double totalTime = sdkGetTimerValue(&totalTimer); + printf("Total time: %f ms \n", totalTime); + if (logDir != NULL) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << totalTime << std::endl; + outFile.close(); + } + + sdkDeleteTimer(&totalTimer); + + exit(EXIT_SUCCESS); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/fastWalshTransform.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/fastWalshTransform.cu new file mode 100644 index 0000000..b65f0b1 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/fastWalshTransform.cu @@ -0,0 +1,242 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Walsh transforms belong to a class of generalized Fourier transformations. + * They have applications in various fields of electrical engineering + * and numeric theory. In this sample we demonstrate efficient implementation + * of naturally-ordered Walsh transform + * (also known as Walsh-Hadamard or Hadamard transform) in CUDA and its + * particular application to dyadic convolution computation. + * Refer to excellent Jorg Arndt's "Algorithms for Programmers" textbook + * http://www.jjj.de/fxt/fxtbook.pdf (Chapter 22) + * + * Victor Podlozhnyuk (vpodlozhnyuk@nvidia.com) + */ + +#include +#include +#include +#include +#include +#include + +//////////////////////////////////////////////////////////////////////////////// +// Reference CPU FWT +//////////////////////////////////////////////////////////////////////////////// +void fwtCPU(float *h_Output, float *h_Input, int log2N) { + const int N = 1 << log2N; + + for (int pos = 0; pos < N; pos++) h_Output[pos] = h_Input[pos]; + + // Cycle through stages with different butterfly strides + for (int stride = N / 2; stride >= 1; stride >>= 1) { + // Cycle through subvectors of (2 * stride) elements + for (int base = 0; base < N; base += 2 * stride) + + // Butterfly index within subvector of (2 * stride) size + for (int j = 0; j < stride; j++) { + int i0 = base + j + 0; + int i1 = base + j + stride; + + float T1 = h_Output[i0]; + float T2 = h_Output[i1]; + h_Output[i0] = T1 + T2; + h_Output[i1] = T1 - T2; + } + } +} + +void slowWTcpu(float *h_Output, float *h_Input, int log2N) { + const int N = 1 << log2N; + + for (int i = 0; i < N; i++) { + double sum = 0; + + for (int j = 0; j < N; j++) { + // Walsh-Hadamard quotient + double q = 1.0; + + for (int t = i & j; t != 0; t >>= 1) + if (t & 1) q = -q; + + sum += q * h_Input[j]; + } + + h_Output[i] = (float)sum; + } +} + +void dyadicConvolutionCPU(float *h_Result, float *h_Data, + float *h_Kernel, int log2dataN, + int log2kernelN) { + const int dataN = 1 << log2dataN; + const int kernelN = 1 << log2kernelN; + + for (int i = 0; i < dataN; i++) { + double sum = 0; + + for (int j = 0; j < kernelN; j++) sum += h_Data[i ^ j] * h_Kernel[j]; + + h_Result[i] = (float)sum; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// GPU FWT +//////////////////////////////////////////////////////////////////////////////// +#include "./include/fastWalshTransform_kernel.cuh" + +//////////////////////////////////////////////////////////////////////////////// +// Data configuration +//////////////////////////////////////////////////////////////////////////////// +const int log2Kernel = 7; +const int log2Data = 23; + +const int dataN = 1 << log2Data; +const int kernelN = 1 << log2Kernel; + +const int DATA_SIZE = dataN * sizeof(float); +const int KERNEL_SIZE = kernelN * sizeof(float); + +const double NOPS = 3.0 * (double)dataN * (double)log2Data / 2.0; + +StopWatchInterface *totalTimer = NULL; +//////////////////////////////////////////////////////////////////////////////// +// Main program +//////////////////////////////////////////////////////////////////////////////// +int main(int argc, char *argv[]) { + + char* logDir = NULL; + if (checkCmdLineFlag(argc, (const char **)argv, "logDir")) { + getCmdLineArgumentString(argc, (const char **)argv, "logDir=", &logDir); + } + + sdkCreateTimer(&totalTimer); + sdkStartTimer(&totalTimer); + float *h_Data, *h_Kernel, *h_ResultCPU, *h_ResultGPU; + + float *d_Data, *d_Kernel; + + double delta, ref, sum_delta2, sum_ref2, L2norm, gpuTime; + + StopWatchInterface *hTimer = NULL; + int i; + + printf("%s Starting...\n\n", argv[0]); + + // use command-line specified CUDA device, otherwise use device with highest + // Gflops/s + findCudaDevice(argc, (const char **)argv); + + sdkCreateTimer(&hTimer); + + printf("Initializing data...\n"); + printf("...allocating CPU memory\n"); + h_Kernel = (float *)malloc(KERNEL_SIZE); + h_Data = (float *)malloc(DATA_SIZE); + h_ResultCPU = (float *)malloc(DATA_SIZE); + h_ResultGPU = (float *)malloc(DATA_SIZE); + printf("...allocating GPU memory\n"); + checkCudaErrors(cudaMalloc((void **)&d_Kernel, DATA_SIZE)); + checkCudaErrors(cudaMalloc((void **)&d_Data, DATA_SIZE)); + + printf("...generating data\n"); + printf("Data length: %i; kernel length: %i\n", dataN, kernelN); + srand(2007); + + for (i = 0; i < kernelN; i++) { + h_Kernel[i] = (float)rand() / (float)RAND_MAX; + } + + for (i = 0; i < dataN; i++) { + h_Data[i] = (float)rand() / (float)RAND_MAX; + } + + checkCudaErrors(cudaMemset(d_Kernel, 0, DATA_SIZE)); + checkCudaErrors( + cudaMemcpy(d_Kernel, h_Kernel, KERNEL_SIZE, cudaMemcpyHostToDevice)); + checkCudaErrors( + cudaMemcpy(d_Data, h_Data, DATA_SIZE, cudaMemcpyHostToDevice)); + + printf("Running GPU dyadic convolution using Fast Walsh Transform...\n"); + checkCudaErrors(cudaDeviceSynchronize()); + sdkResetTimer(&hTimer); + sdkStartTimer(&hTimer); + fwtBatchGPU(d_Data, 1, log2Data); + fwtBatchGPU(d_Kernel, 1, log2Data); + modulateGPU(d_Data, d_Kernel, dataN); + fwtBatchGPU(d_Data, 1, log2Data); + checkCudaErrors(cudaDeviceSynchronize()); + sdkStopTimer(&hTimer); + gpuTime = sdkGetTimerValue(&hTimer); + printf("GPU time: %f ms; GOP/s: %f\n", gpuTime, + NOPS / (gpuTime * 0.001 * 1E+9)); + + printf("Reading back GPU results...\n"); + checkCudaErrors( + cudaMemcpy(h_ResultGPU, d_Data, DATA_SIZE, cudaMemcpyDeviceToHost)); + + printf("Running straightforward CPU dyadic convolution...\n"); + dyadicConvolutionCPU(h_ResultCPU, h_Data, h_Kernel, log2Data, log2Kernel); + + printf("Comparing the results...\n"); + sum_delta2 = 0; + sum_ref2 = 0; + + for (i = 0; i < dataN; i++) { + delta = h_ResultCPU[i] - h_ResultGPU[i]; + ref = h_ResultCPU[i]; + sum_delta2 += delta * delta; + sum_ref2 += ref * ref; + } + + L2norm = sqrt(sum_delta2 / sum_ref2); + + printf("Shutting down...\n"); + sdkDeleteTimer(&hTimer); + checkCudaErrors(cudaFree(d_Data)); + checkCudaErrors(cudaFree(d_Kernel)); + free(h_ResultGPU); + free(h_ResultCPU); + free(h_Data); + free(h_Kernel); + + printf("L2 norm: %E\n", L2norm); + printf(L2norm < 1e-6 ? "Test passed\n" : "Test failed!\n"); + + sdkStopTimer(&totalTimer); + double totalTime = sdkGetTimerValue(&totalTimer); + printf("Total time: %f ms \n", totalTime); + if (logDir != NULL) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << totalTime << std::endl; + outFile.close(); + } + + sdkDeleteTimer(&totalTimer); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/BlackScholes_kernel.cuh b/GPU-Virtual-Service/gpu-remoting/src/sample/include/BlackScholes_kernel.cuh new file mode 100644 index 0000000..b7cfccc --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/BlackScholes_kernel.cuh @@ -0,0 +1,167 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +//////////////////////////////////////////////////////////////////////////////// +// Polynomial approximation of cumulative normal distribution function +//////////////////////////////////////////////////////////////////////////////// +static double CND(double d) { + const double A1 = 0.31938153; + const double A2 = -0.356563782; + const double A3 = 1.781477937; + const double A4 = -1.821255978; + const double A5 = 1.330274429; + const double RSQRT2PI = 0.39894228040143267793994605993438; + + double K = 1.0 / (1.0 + 0.2316419 * fabs(d)); + + double cnd = RSQRT2PI * exp(-0.5 * d * d) * + (K * (A1 + K * (A2 + K * (A3 + K * (A4 + K * A5))))); + + if (d > 0) cnd = 1.0 - cnd; + + return cnd; +} + +//////////////////////////////////////////////////////////////////////////////// +// Black-Scholes formula for both call and put +//////////////////////////////////////////////////////////////////////////////// +static void BlackScholesBodyCPU(float &callResult, float &putResult, + float Sf, // Stock price + float Xf, // Option strike + float Tf, // Option years + float Rf, // Riskless rate + float Vf // Volatility rate + ) { + double S = Sf, X = Xf, T = Tf, R = Rf, V = Vf; + + double sqrtT = sqrt(T); + double d1 = (log(S / X) + (R + 0.5 * V * V) * T) / (V * sqrtT); + double d2 = d1 - V * sqrtT; + double CNDD1 = CND(d1); + double CNDD2 = CND(d2); + + // Calculate Call and Put simultaneously + double expRT = exp(-R * T); + callResult = (float)(S * CNDD1 - X * expRT * CNDD2); + putResult = (float)(X * expRT * (1.0 - CNDD2) - S * (1.0 - CNDD1)); +} + +//////////////////////////////////////////////////////////////////////////////// +// Process an array of optN options +//////////////////////////////////////////////////////////////////////////////// +extern "C" void BlackScholesCPU(float *h_CallResult, float *h_PutResult, + float *h_StockPrice, float *h_OptionStrike, + float *h_OptionYears, float Riskfree, + float Volatility, int optN) { + for (int opt = 0; opt < optN; opt++) + BlackScholesBodyCPU(h_CallResult[opt], h_PutResult[opt], h_StockPrice[opt], + h_OptionStrike[opt], h_OptionYears[opt], Riskfree, + Volatility); +} + + +//////////////////////////////////////////////////////////////////////////////// +// Polynomial approximation of cumulative normal distribution function +//////////////////////////////////////////////////////////////////////////////// +__device__ inline float cndGPU(float d) { + const float A1 = 0.31938153f; + const float A2 = -0.356563782f; + const float A3 = 1.781477937f; + const float A4 = -1.821255978f; + const float A5 = 1.330274429f; + const float RSQRT2PI = 0.39894228040143267793994605993438f; + + float K = __fdividef(1.0f, (1.0f + 0.2316419f * fabsf(d))); + + float cnd = RSQRT2PI * __expf(-0.5f * d * d) * + (K * (A1 + K * (A2 + K * (A3 + K * (A4 + K * A5))))); + + if (d > 0) cnd = 1.0f - cnd; + + return cnd; +} + +//////////////////////////////////////////////////////////////////////////////// +// Black-Scholes formula for both call and put +//////////////////////////////////////////////////////////////////////////////// +__device__ inline void BlackScholesBodyGPU(float &CallResult, float &PutResult, + float S, // Stock price + float X, // Option strike + float T, // Option years + float R, // Riskless rate + float V // Volatility rate + ) { + float sqrtT, expRT; + float d1, d2, CNDD1, CNDD2; + + sqrtT = __fdividef(1.0F, rsqrtf(T)); + d1 = __fdividef(__logf(S / X) + (R + 0.5f * V * V) * T, V * sqrtT); + d2 = d1 - V * sqrtT; + + CNDD1 = cndGPU(d1); + CNDD2 = cndGPU(d2); + + // Calculate Call and Put simultaneously + expRT = __expf(-R * T); + CallResult = S * CNDD1 - X * expRT * CNDD2; + PutResult = X * expRT * (1.0f - CNDD2) - S * (1.0f - CNDD1); +} + +//////////////////////////////////////////////////////////////////////////////// +// Process an array of optN options on GPU +//////////////////////////////////////////////////////////////////////////////// +__launch_bounds__(128) __global__ + void BlackScholesGPU(float2 *__restrict d_CallResult, + float2 *__restrict d_PutResult, + float2 *__restrict d_StockPrice, + float2 *__restrict d_OptionStrike, + float2 *__restrict d_OptionYears, float Riskfree, + float Volatility, int optN) { + ////Thread index + // const int tid = blockDim.x * blockIdx.x + threadIdx.x; + ////Total number of threads in execution grid + // const int THREAD_N = blockDim.x * gridDim.x; + + const int opt = blockDim.x * blockIdx.x + threadIdx.x; + + // Calculating 2 options per thread to increase ILP (instruction level + // parallelism) + if (opt < (optN / 2)) { + float callResult1, callResult2; + float putResult1, putResult2; + BlackScholesBodyGPU(callResult1, putResult1, d_StockPrice[opt].x, + d_OptionStrike[opt].x, d_OptionYears[opt].x, Riskfree, + Volatility); + BlackScholesBodyGPU(callResult2, putResult2, d_StockPrice[opt].y, + d_OptionStrike[opt].y, d_OptionYears[opt].y, Riskfree, + Volatility); + d_CallResult[opt] = make_float2(callResult1, callResult2); + d_PutResult[opt] = make_float2(putResult1, putResult2); + } +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/convolutionSeparable_common.cuh b/GPU-Virtual-Service/gpu-remoting/src/sample/include/convolutionSeparable_common.cuh new file mode 100644 index 0000000..f968521 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/convolutionSeparable_common.cuh @@ -0,0 +1,268 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef CONVOLUTIONSEPARABLE_COMMON_H +#define CONVOLUTIONSEPARABLE_COMMON_H + +#include +#include +#include + +namespace cg = cooperative_groups; + +#define KERNEL_RADIUS 8 +#define KERNEL_LENGTH (2 * KERNEL_RADIUS + 1) + + + +//////////////////////////////////////////////////////////////////////////////// +// Reference row convolution filter +//////////////////////////////////////////////////////////////////////////////// +void convolutionRowCPU(float *h_Dst, float *h_Src, float *h_Kernel, + int imageW, int imageH, int kernelR) { + for (int y = 0; y < imageH; y++) + for (int x = 0; x < imageW; x++) { + float sum = 0; + + for (int k = -kernelR; k <= kernelR; k++) { + int d = x + k; + + if (d >= 0 && d < imageW) + sum += h_Src[y * imageW + d] * h_Kernel[kernelR - k]; + } + + h_Dst[y * imageW + x] = sum; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Reference column convolution filter +//////////////////////////////////////////////////////////////////////////////// +void convolutionColumnCPU(float *h_Dst, float *h_Src, + float *h_Kernel, int imageW, int imageH, + int kernelR) { + for (int y = 0; y < imageH; y++) + for (int x = 0; x < imageW; x++) { + float sum = 0; + + for (int k = -kernelR; k <= kernelR; k++) { + int d = y + k; + + if (d >= 0 && d < imageH) + sum += h_Src[d * imageW + x] * h_Kernel[kernelR - k]; + } + + h_Dst[y * imageW + x] = sum; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// GPU convolution +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +// Convolution kernel storage +//////////////////////////////////////////////////////////////////////////////// +__constant__ float c_Kernel[KERNEL_LENGTH]; + +void setConvolutionKernel(float *h_Kernel) { + cudaMemcpyToSymbol(c_Kernel, h_Kernel, KERNEL_LENGTH * sizeof(float)); +} + +//////////////////////////////////////////////////////////////////////////////// +// Row convolution filter +//////////////////////////////////////////////////////////////////////////////// +#define ROWS_BLOCKDIM_X 16 +#define ROWS_BLOCKDIM_Y 4 +#define ROWS_RESULT_STEPS 8 +#define ROWS_HALO_STEPS 1 + +__global__ void convolutionRowsKernel(float *d_Dst, float *d_Src, int imageW, + int imageH, int pitch) { + // Handle to thread block group + cg::thread_block cta = cg::this_thread_block(); + __shared__ float + s_Data[ROWS_BLOCKDIM_Y][(ROWS_RESULT_STEPS + 2 * ROWS_HALO_STEPS) * + ROWS_BLOCKDIM_X]; + + // Offset to the left halo edge + const int baseX = + (blockIdx.x * ROWS_RESULT_STEPS - ROWS_HALO_STEPS) * ROWS_BLOCKDIM_X + + threadIdx.x; + const int baseY = blockIdx.y * ROWS_BLOCKDIM_Y + threadIdx.y; + + d_Src += baseY * pitch + baseX; + d_Dst += baseY * pitch + baseX; + +// Load main data +#pragma unroll + + for (int i = ROWS_HALO_STEPS; i < ROWS_HALO_STEPS + ROWS_RESULT_STEPS; i++) { + s_Data[threadIdx.y][threadIdx.x + i * ROWS_BLOCKDIM_X] = + d_Src[i * ROWS_BLOCKDIM_X]; + } + +// Load left halo +#pragma unroll + + for (int i = 0; i < ROWS_HALO_STEPS; i++) { + s_Data[threadIdx.y][threadIdx.x + i * ROWS_BLOCKDIM_X] = + (baseX >= -i * ROWS_BLOCKDIM_X) ? d_Src[i * ROWS_BLOCKDIM_X] : 0; + } + +// Load right halo +#pragma unroll + + for (int i = ROWS_HALO_STEPS + ROWS_RESULT_STEPS; + i < ROWS_HALO_STEPS + ROWS_RESULT_STEPS + ROWS_HALO_STEPS; i++) { + s_Data[threadIdx.y][threadIdx.x + i * ROWS_BLOCKDIM_X] = + (imageW - baseX > i * ROWS_BLOCKDIM_X) ? d_Src[i * ROWS_BLOCKDIM_X] : 0; + } + + // Compute and store results + cg::sync(cta); +#pragma unroll + + for (int i = ROWS_HALO_STEPS; i < ROWS_HALO_STEPS + ROWS_RESULT_STEPS; i++) { + float sum = 0; + +#pragma unroll + + for (int j = -KERNEL_RADIUS; j <= KERNEL_RADIUS; j++) { + sum += c_Kernel[KERNEL_RADIUS - j] * + s_Data[threadIdx.y][threadIdx.x + i * ROWS_BLOCKDIM_X + j]; + } + + d_Dst[i * ROWS_BLOCKDIM_X] = sum; + } +} + +void convolutionRowsGPU(float *d_Dst, float *d_Src, int imageW, + int imageH) { + assert(ROWS_BLOCKDIM_X * ROWS_HALO_STEPS >= KERNEL_RADIUS); + assert(imageW % (ROWS_RESULT_STEPS * ROWS_BLOCKDIM_X) == 0); + assert(imageH % ROWS_BLOCKDIM_Y == 0); + + dim3 blocks(imageW / (ROWS_RESULT_STEPS * ROWS_BLOCKDIM_X), + imageH / ROWS_BLOCKDIM_Y); + dim3 threads(ROWS_BLOCKDIM_X, ROWS_BLOCKDIM_Y); + + convolutionRowsKernel<<>>(d_Dst, d_Src, imageW, imageH, + imageW); + getLastCudaError("convolutionRowsKernel() execution failed\n"); +} + +//////////////////////////////////////////////////////////////////////////////// +// Column convolution filter +//////////////////////////////////////////////////////////////////////////////// +#define COLUMNS_BLOCKDIM_X 16 +#define COLUMNS_BLOCKDIM_Y 8 +#define COLUMNS_RESULT_STEPS 8 +#define COLUMNS_HALO_STEPS 1 + +__global__ void convolutionColumnsKernel(float *d_Dst, float *d_Src, int imageW, + int imageH, int pitch) { + // Handle to thread block group + cg::thread_block cta = cg::this_thread_block(); + __shared__ float s_Data[COLUMNS_BLOCKDIM_X][(COLUMNS_RESULT_STEPS + + 2 * COLUMNS_HALO_STEPS) * + COLUMNS_BLOCKDIM_Y + + 1]; + + // Offset to the upper halo edge + const int baseX = blockIdx.x * COLUMNS_BLOCKDIM_X + threadIdx.x; + const int baseY = (blockIdx.y * COLUMNS_RESULT_STEPS - COLUMNS_HALO_STEPS) * + COLUMNS_BLOCKDIM_Y + + threadIdx.y; + d_Src += baseY * pitch + baseX; + d_Dst += baseY * pitch + baseX; + +// Main data +#pragma unroll + + for (int i = COLUMNS_HALO_STEPS; + i < COLUMNS_HALO_STEPS + COLUMNS_RESULT_STEPS; i++) { + s_Data[threadIdx.x][threadIdx.y + i * COLUMNS_BLOCKDIM_Y] = + d_Src[i * COLUMNS_BLOCKDIM_Y * pitch]; + } + +// Upper halo +#pragma unroll + + for (int i = 0; i < COLUMNS_HALO_STEPS; i++) { + s_Data[threadIdx.x][threadIdx.y + i * COLUMNS_BLOCKDIM_Y] = + (baseY >= -i * COLUMNS_BLOCKDIM_Y) + ? d_Src[i * COLUMNS_BLOCKDIM_Y * pitch] + : 0; + } + +// Lower halo +#pragma unroll + + for (int i = COLUMNS_HALO_STEPS + COLUMNS_RESULT_STEPS; + i < COLUMNS_HALO_STEPS + COLUMNS_RESULT_STEPS + COLUMNS_HALO_STEPS; + i++) { + s_Data[threadIdx.x][threadIdx.y + i * COLUMNS_BLOCKDIM_Y] = + (imageH - baseY > i * COLUMNS_BLOCKDIM_Y) + ? d_Src[i * COLUMNS_BLOCKDIM_Y * pitch] + : 0; + } + + // Compute and store results + cg::sync(cta); +#pragma unroll + + for (int i = COLUMNS_HALO_STEPS; + i < COLUMNS_HALO_STEPS + COLUMNS_RESULT_STEPS; i++) { + float sum = 0; +#pragma unroll + + for (int j = -KERNEL_RADIUS; j <= KERNEL_RADIUS; j++) { + sum += c_Kernel[KERNEL_RADIUS - j] * + s_Data[threadIdx.x][threadIdx.y + i * COLUMNS_BLOCKDIM_Y + j]; + } + + d_Dst[i * COLUMNS_BLOCKDIM_Y * pitch] = sum; + } +} + +void convolutionColumnsGPU(float *d_Dst, float *d_Src, int imageW, + int imageH) { + assert(COLUMNS_BLOCKDIM_Y * COLUMNS_HALO_STEPS >= KERNEL_RADIUS); + assert(imageW % COLUMNS_BLOCKDIM_X == 0); + assert(imageH % (COLUMNS_RESULT_STEPS * COLUMNS_BLOCKDIM_Y) == 0); + + dim3 blocks(imageW / COLUMNS_BLOCKDIM_X, + imageH / (COLUMNS_RESULT_STEPS * COLUMNS_BLOCKDIM_Y)); + dim3 threads(COLUMNS_BLOCKDIM_X, COLUMNS_BLOCKDIM_Y); + + convolutionColumnsKernel<<>>(d_Dst, d_Src, imageW, imageH, + imageW); + getLastCudaError("convolutionColumnsKernel() execution failed\n"); +} + +#endif diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/exception.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/exception.h new file mode 100644 index 0000000..ca8ac25 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/exception.h @@ -0,0 +1,151 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* CUda UTility Library */ +#ifndef COMMON_EXCEPTION_H_ +#define COMMON_EXCEPTION_H_ + +// includes, system +#include +#include +#include +#include +#include + +//! Exception wrapper. +//! @param Std_Exception Exception out of namespace std for easy typing. +template +class Exception : public Std_Exception { + public: + //! @brief Static construction interface + //! @return Alwayss throws ( Located_Exception) + //! @param file file in which the Exception occurs + //! @param line line in which the Exception occurs + //! @param detailed details on the code fragment causing the Exception + static void throw_it(const char *file, const int line, + const char *detailed = "-"); + + //! Static construction interface + //! @return Alwayss throws ( Located_Exception) + //! @param file file in which the Exception occurs + //! @param line line in which the Exception occurs + //! @param detailed details on the code fragment causing the Exception + static void throw_it(const char *file, const int line, + const std::string &detailed); + + //! Destructor + virtual ~Exception() throw(); + + private: + //! Constructor, default (private) + Exception(); + + //! Constructor, standard + //! @param str string returned by what() + explicit Exception(const std::string &str); +}; + +//////////////////////////////////////////////////////////////////////////////// +//! Exception handler function for arbitrary exceptions +//! @param ex exception to handle +//////////////////////////////////////////////////////////////////////////////// +template +inline void handleException(const Exception_Typ &ex) { + std::cerr << ex.what() << std::endl; + + exit(EXIT_FAILURE); +} + +//! Convenience macros + +//! Exception caused by dynamic program behavior, e.g. file does not exist +#define RUNTIME_EXCEPTION(msg) \ + Exception::throw_it(__FILE__, __LINE__, msg) + +//! Logic exception in program, e.g. an assert failed +#define LOGIC_EXCEPTION(msg) \ + Exception::throw_it(__FILE__, __LINE__, msg) + +//! Out of range exception +#define RANGE_EXCEPTION(msg) \ + Exception::throw_it(__FILE__, __LINE__, msg) + +//////////////////////////////////////////////////////////////////////////////// +//! Implementation + +// includes, system +#include + +//////////////////////////////////////////////////////////////////////////////// +//! Static construction interface. +//! @param Exception causing code fragment (file and line) and detailed infos. +//////////////////////////////////////////////////////////////////////////////// +/*static*/ template +void Exception::throw_it(const char *file, const int line, + const char *detailed) { + std::stringstream s; + + // Quiet heavy-weight but exceptions are not for + // performance / release versions + s << "Exception in file '" << file << "' in line " << line << "\n" + << "Detailed description: " << detailed << "\n"; + + throw Exception(s.str()); +} + +//////////////////////////////////////////////////////////////////////////////// +//! Static construction interface. +//! @param Exception causing code fragment (file and line) and detailed infos. +//////////////////////////////////////////////////////////////////////////////// +/*static*/ template +void Exception::throw_it(const char *file, const int line, + const std::string &msg) { + throw_it(file, line, msg.c_str()); +} + +//////////////////////////////////////////////////////////////////////////////// +//! Constructor, default (private). +//////////////////////////////////////////////////////////////////////////////// +template +Exception::Exception() : Std_Exception("Unknown Exception.\n") {} + +//////////////////////////////////////////////////////////////////////////////// +//! Constructor, standard (private). +//! String returned by what(). +//////////////////////////////////////////////////////////////////////////////// +template +Exception::Exception(const std::string &s) : Std_Exception(s) {} + +//////////////////////////////////////////////////////////////////////////////// +//! Destructor +//////////////////////////////////////////////////////////////////////////////// +template +Exception::~Exception() throw() {} + + // functions, exported + +#endif // COMMON_EXCEPTION_H_ diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/fastWalshTransform_kernel.cuh b/GPU-Virtual-Service/gpu-remoting/src/sample/include/fastWalshTransform_kernel.cuh new file mode 100644 index 0000000..4f64117 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/fastWalshTransform_kernel.cuh @@ -0,0 +1,185 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef FWT_KERNEL_CUH +#define FWT_KERNEL_CUH +#ifndef fwt_kernel_cuh +#define fwt_kernel_cuh + +#include + +namespace cg = cooperative_groups; + +/////////////////////////////////////////////////////////////////////////////// +// Elementary(for vectors less than elementary size) in-shared memory +// combined radix-2 + radix-4 Fast Walsh Transform +/////////////////////////////////////////////////////////////////////////////// +#define ELEMENTARY_LOG2SIZE 11 + +__global__ void fwtBatch1Kernel(float *d_Output, float *d_Input, int log2N) { + // Handle to thread block group + cg::thread_block cta = cg::this_thread_block(); + const int N = 1 << log2N; + const int base = blockIdx.x << log2N; + + //(2 ** 11) * 4 bytes == 8KB -- maximum s_data[] size for G80 + extern __shared__ float s_data[]; + float *d_Src = d_Input + base; + float *d_Dst = d_Output + base; + + for (int pos = threadIdx.x; pos < N; pos += blockDim.x) { + s_data[pos] = d_Src[pos]; + } + + // Main radix-4 stages + const int pos = threadIdx.x; + + for (int stride = N >> 2; stride > 0; stride >>= 2) { + int lo = pos & (stride - 1); + int i0 = ((pos - lo) << 2) + lo; + int i1 = i0 + stride; + int i2 = i1 + stride; + int i3 = i2 + stride; + + cg::sync(cta); + float D0 = s_data[i0]; + float D1 = s_data[i1]; + float D2 = s_data[i2]; + float D3 = s_data[i3]; + + float T; + T = D0; + D0 = D0 + D2; + D2 = T - D2; + T = D1; + D1 = D1 + D3; + D3 = T - D3; + T = D0; + s_data[i0] = D0 + D1; + s_data[i1] = T - D1; + T = D2; + s_data[i2] = D2 + D3; + s_data[i3] = T - D3; + } + + // Do single radix-2 stage for odd power of two + if (log2N & 1) { + cg::sync(cta); + + for (int pos = threadIdx.x; pos < N / 2; pos += blockDim.x) { + int i0 = pos << 1; + int i1 = i0 + 1; + + float D0 = s_data[i0]; + float D1 = s_data[i1]; + s_data[i0] = D0 + D1; + s_data[i1] = D0 - D1; + } + } + + cg::sync(cta); + + for (int pos = threadIdx.x; pos < N; pos += blockDim.x) { + d_Dst[pos] = s_data[pos]; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Single in-global memory radix-4 Fast Walsh Transform pass +// (for strides exceeding elementary vector size) +//////////////////////////////////////////////////////////////////////////////// +__global__ void fwtBatch2Kernel(float *d_Output, float *d_Input, int stride) { + const int pos = blockIdx.x * blockDim.x + threadIdx.x; + const int N = blockDim.x * gridDim.x * 4; + + float *d_Src = d_Input + blockIdx.y * N; + float *d_Dst = d_Output + blockIdx.y * N; + + int lo = pos & (stride - 1); + int i0 = ((pos - lo) << 2) + lo; + int i1 = i0 + stride; + int i2 = i1 + stride; + int i3 = i2 + stride; + + float D0 = d_Src[i0]; + float D1 = d_Src[i1]; + float D2 = d_Src[i2]; + float D3 = d_Src[i3]; + + float T; + T = D0; + D0 = D0 + D2; + D2 = T - D2; + T = D1; + D1 = D1 + D3; + D3 = T - D3; + T = D0; + d_Dst[i0] = D0 + D1; + d_Dst[i1] = T - D1; + T = D2; + d_Dst[i2] = D2 + D3; + d_Dst[i3] = T - D3; +} + +//////////////////////////////////////////////////////////////////////////////// +// Put everything together: batched Fast Walsh Transform CPU front-end +//////////////////////////////////////////////////////////////////////////////// +void fwtBatchGPU(float *d_Data, int M, int log2N) { + const int THREAD_N = 256; + + int N = 1 << log2N; + dim3 grid((1 << log2N) / (4 * THREAD_N), M, 1); + + for (; log2N > ELEMENTARY_LOG2SIZE; log2N -= 2, N >>= 2, M <<= 2) { + fwtBatch2Kernel<<>>(d_Data, d_Data, N / 4); + getLastCudaError("fwtBatch2Kernel() execution failed\n"); + } + + fwtBatch1Kernel<<>>(d_Data, d_Data, log2N); + getLastCudaError("fwtBatch1Kernel() execution failed\n"); +} + +//////////////////////////////////////////////////////////////////////////////// +// Modulate two arrays +//////////////////////////////////////////////////////////////////////////////// +__global__ void modulateKernel(float *d_A, float *d_B, int N) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + int numThreads = blockDim.x * gridDim.x; + float rcpN = 1.0f / (float)N; + + for (int pos = tid; pos < N; pos += numThreads) { + d_A[pos] *= d_B[pos] * rcpN; + } +} + +// Interface to modulateKernel() +void modulateGPU(float *d_A, float *d_B, int N) { + modulateKernel<<<128, 256>>>(d_A, d_B, N); +} + +#endif +#endif diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_cuda.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_cuda.h new file mode 100644 index 0000000..441b88d --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_cuda.h @@ -0,0 +1,976 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +//////////////////////////////////////////////////////////////////////////////// +// These are CUDA Helper functions for initialization and error checking + +#ifndef COMMON_HELPER_CUDA_H_ +#define COMMON_HELPER_CUDA_H_ + +#pragma once + +#include +#include +#include +#include + +#include + +#ifndef EXIT_WAIVED +#define EXIT_WAIVED 2 +#endif + +// Note, it is required that your SDK sample to include the proper header +// files, please refer the CUDA examples for examples of the needed CUDA +// headers, which may change depending on which CUDA functions are used. + +// CUDA Runtime error messages +#ifdef __DRIVER_TYPES_H__ +static const char *_cudaGetErrorEnum(cudaError_t error) { + return cudaGetErrorName(error); +} +#endif + +#ifdef CUDA_DRIVER_API +// CUDA Driver API errors +static const char *_cudaGetErrorEnum(CUresult error) { + static char unknown[] = ""; + const char *ret = NULL; + cuGetErrorName(error, &ret); + return ret ? ret : unknown; +} +#endif + +#ifdef CUBLAS_API_H_ +// cuBLAS API errors +static const char *_cudaGetErrorEnum(cublasStatus_t error) { + switch (error) { + case CUBLAS_STATUS_SUCCESS: + return "CUBLAS_STATUS_SUCCESS"; + + case CUBLAS_STATUS_NOT_INITIALIZED: + return "CUBLAS_STATUS_NOT_INITIALIZED"; + + case CUBLAS_STATUS_ALLOC_FAILED: + return "CUBLAS_STATUS_ALLOC_FAILED"; + + case CUBLAS_STATUS_INVALID_VALUE: + return "CUBLAS_STATUS_INVALID_VALUE"; + + case CUBLAS_STATUS_ARCH_MISMATCH: + return "CUBLAS_STATUS_ARCH_MISMATCH"; + + case CUBLAS_STATUS_MAPPING_ERROR: + return "CUBLAS_STATUS_MAPPING_ERROR"; + + case CUBLAS_STATUS_EXECUTION_FAILED: + return "CUBLAS_STATUS_EXECUTION_FAILED"; + + case CUBLAS_STATUS_INTERNAL_ERROR: + return "CUBLAS_STATUS_INTERNAL_ERROR"; + + case CUBLAS_STATUS_NOT_SUPPORTED: + return "CUBLAS_STATUS_NOT_SUPPORTED"; + + case CUBLAS_STATUS_LICENSE_ERROR: + return "CUBLAS_STATUS_LICENSE_ERROR"; + } + + return ""; +} +#endif + +#ifdef _CUFFT_H_ +// cuFFT API errors +static const char *_cudaGetErrorEnum(cufftResult error) { + switch (error) { + case CUFFT_SUCCESS: + return "CUFFT_SUCCESS"; + + case CUFFT_INVALID_PLAN: + return "CUFFT_INVALID_PLAN"; + + case CUFFT_ALLOC_FAILED: + return "CUFFT_ALLOC_FAILED"; + + case CUFFT_INVALID_TYPE: + return "CUFFT_INVALID_TYPE"; + + case CUFFT_INVALID_VALUE: + return "CUFFT_INVALID_VALUE"; + + case CUFFT_INTERNAL_ERROR: + return "CUFFT_INTERNAL_ERROR"; + + case CUFFT_EXEC_FAILED: + return "CUFFT_EXEC_FAILED"; + + case CUFFT_SETUP_FAILED: + return "CUFFT_SETUP_FAILED"; + + case CUFFT_INVALID_SIZE: + return "CUFFT_INVALID_SIZE"; + + case CUFFT_UNALIGNED_DATA: + return "CUFFT_UNALIGNED_DATA"; + + case CUFFT_INCOMPLETE_PARAMETER_LIST: + return "CUFFT_INCOMPLETE_PARAMETER_LIST"; + + case CUFFT_INVALID_DEVICE: + return "CUFFT_INVALID_DEVICE"; + + case CUFFT_PARSE_ERROR: + return "CUFFT_PARSE_ERROR"; + + case CUFFT_NO_WORKSPACE: + return "CUFFT_NO_WORKSPACE"; + + case CUFFT_NOT_IMPLEMENTED: + return "CUFFT_NOT_IMPLEMENTED"; + + case CUFFT_LICENSE_ERROR: + return "CUFFT_LICENSE_ERROR"; + + case CUFFT_NOT_SUPPORTED: + return "CUFFT_NOT_SUPPORTED"; + } + + return ""; +} +#endif + +#ifdef CUSPARSEAPI +// cuSPARSE API errors +static const char *_cudaGetErrorEnum(cusparseStatus_t error) { + switch (error) { + case CUSPARSE_STATUS_SUCCESS: + return "CUSPARSE_STATUS_SUCCESS"; + + case CUSPARSE_STATUS_NOT_INITIALIZED: + return "CUSPARSE_STATUS_NOT_INITIALIZED"; + + case CUSPARSE_STATUS_ALLOC_FAILED: + return "CUSPARSE_STATUS_ALLOC_FAILED"; + + case CUSPARSE_STATUS_INVALID_VALUE: + return "CUSPARSE_STATUS_INVALID_VALUE"; + + case CUSPARSE_STATUS_ARCH_MISMATCH: + return "CUSPARSE_STATUS_ARCH_MISMATCH"; + + case CUSPARSE_STATUS_MAPPING_ERROR: + return "CUSPARSE_STATUS_MAPPING_ERROR"; + + case CUSPARSE_STATUS_EXECUTION_FAILED: + return "CUSPARSE_STATUS_EXECUTION_FAILED"; + + case CUSPARSE_STATUS_INTERNAL_ERROR: + return "CUSPARSE_STATUS_INTERNAL_ERROR"; + + case CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED: + return "CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED"; + } + + return ""; +} +#endif + +#ifdef CUSOLVER_COMMON_H_ +// cuSOLVER API errors +static const char *_cudaGetErrorEnum(cusolverStatus_t error) { + switch (error) { + case CUSOLVER_STATUS_SUCCESS: + return "CUSOLVER_STATUS_SUCCESS"; + case CUSOLVER_STATUS_NOT_INITIALIZED: + return "CUSOLVER_STATUS_NOT_INITIALIZED"; + case CUSOLVER_STATUS_ALLOC_FAILED: + return "CUSOLVER_STATUS_ALLOC_FAILED"; + case CUSOLVER_STATUS_INVALID_VALUE: + return "CUSOLVER_STATUS_INVALID_VALUE"; + case CUSOLVER_STATUS_ARCH_MISMATCH: + return "CUSOLVER_STATUS_ARCH_MISMATCH"; + case CUSOLVER_STATUS_MAPPING_ERROR: + return "CUSOLVER_STATUS_MAPPING_ERROR"; + case CUSOLVER_STATUS_EXECUTION_FAILED: + return "CUSOLVER_STATUS_EXECUTION_FAILED"; + case CUSOLVER_STATUS_INTERNAL_ERROR: + return "CUSOLVER_STATUS_INTERNAL_ERROR"; + case CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED: + return "CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED"; + case CUSOLVER_STATUS_NOT_SUPPORTED: + return "CUSOLVER_STATUS_NOT_SUPPORTED "; + case CUSOLVER_STATUS_ZERO_PIVOT: + return "CUSOLVER_STATUS_ZERO_PIVOT"; + case CUSOLVER_STATUS_INVALID_LICENSE: + return "CUSOLVER_STATUS_INVALID_LICENSE"; + } + + return ""; +} +#endif + +#ifdef CURAND_H_ +// cuRAND API errors +static const char *_cudaGetErrorEnum(curandStatus_t error) { + switch (error) { + case CURAND_STATUS_SUCCESS: + return "CURAND_STATUS_SUCCESS"; + + case CURAND_STATUS_VERSION_MISMATCH: + return "CURAND_STATUS_VERSION_MISMATCH"; + + case CURAND_STATUS_NOT_INITIALIZED: + return "CURAND_STATUS_NOT_INITIALIZED"; + + case CURAND_STATUS_ALLOCATION_FAILED: + return "CURAND_STATUS_ALLOCATION_FAILED"; + + case CURAND_STATUS_TYPE_ERROR: + return "CURAND_STATUS_TYPE_ERROR"; + + case CURAND_STATUS_OUT_OF_RANGE: + return "CURAND_STATUS_OUT_OF_RANGE"; + + case CURAND_STATUS_LENGTH_NOT_MULTIPLE: + return "CURAND_STATUS_LENGTH_NOT_MULTIPLE"; + + case CURAND_STATUS_DOUBLE_PRECISION_REQUIRED: + return "CURAND_STATUS_DOUBLE_PRECISION_REQUIRED"; + + case CURAND_STATUS_LAUNCH_FAILURE: + return "CURAND_STATUS_LAUNCH_FAILURE"; + + case CURAND_STATUS_PREEXISTING_FAILURE: + return "CURAND_STATUS_PREEXISTING_FAILURE"; + + case CURAND_STATUS_INITIALIZATION_FAILED: + return "CURAND_STATUS_INITIALIZATION_FAILED"; + + case CURAND_STATUS_ARCH_MISMATCH: + return "CURAND_STATUS_ARCH_MISMATCH"; + + case CURAND_STATUS_INTERNAL_ERROR: + return "CURAND_STATUS_INTERNAL_ERROR"; + } + + return ""; +} +#endif + +#ifdef NVJPEGAPI +// nvJPEG API errors +static const char *_cudaGetErrorEnum(nvjpegStatus_t error) { + switch (error) { + case NVJPEG_STATUS_SUCCESS: + return "NVJPEG_STATUS_SUCCESS"; + + case NVJPEG_STATUS_NOT_INITIALIZED: + return "NVJPEG_STATUS_NOT_INITIALIZED"; + + case NVJPEG_STATUS_INVALID_PARAMETER: + return "NVJPEG_STATUS_INVALID_PARAMETER"; + + case NVJPEG_STATUS_BAD_JPEG: + return "NVJPEG_STATUS_BAD_JPEG"; + + case NVJPEG_STATUS_JPEG_NOT_SUPPORTED: + return "NVJPEG_STATUS_JPEG_NOT_SUPPORTED"; + + case NVJPEG_STATUS_ALLOCATOR_FAILURE: + return "NVJPEG_STATUS_ALLOCATOR_FAILURE"; + + case NVJPEG_STATUS_EXECUTION_FAILED: + return "NVJPEG_STATUS_EXECUTION_FAILED"; + + case NVJPEG_STATUS_ARCH_MISMATCH: + return "NVJPEG_STATUS_ARCH_MISMATCH"; + + case NVJPEG_STATUS_INTERNAL_ERROR: + return "NVJPEG_STATUS_INTERNAL_ERROR"; + } + + return ""; +} +#endif + +#ifdef NV_NPPIDEFS_H +// NPP API errors +static const char *_cudaGetErrorEnum(NppStatus error) { + switch (error) { + case NPP_NOT_SUPPORTED_MODE_ERROR: + return "NPP_NOT_SUPPORTED_MODE_ERROR"; + + case NPP_ROUND_MODE_NOT_SUPPORTED_ERROR: + return "NPP_ROUND_MODE_NOT_SUPPORTED_ERROR"; + + case NPP_RESIZE_NO_OPERATION_ERROR: + return "NPP_RESIZE_NO_OPERATION_ERROR"; + + case NPP_NOT_SUFFICIENT_COMPUTE_CAPABILITY: + return "NPP_NOT_SUFFICIENT_COMPUTE_CAPABILITY"; + +#if ((NPP_VERSION_MAJOR << 12) + (NPP_VERSION_MINOR << 4)) <= 0x5000 + + case NPP_BAD_ARG_ERROR: + return "NPP_BAD_ARGUMENT_ERROR"; + + case NPP_COEFF_ERROR: + return "NPP_COEFFICIENT_ERROR"; + + case NPP_RECT_ERROR: + return "NPP_RECTANGLE_ERROR"; + + case NPP_QUAD_ERROR: + return "NPP_QUADRANGLE_ERROR"; + + case NPP_MEM_ALLOC_ERR: + return "NPP_MEMORY_ALLOCATION_ERROR"; + + case NPP_HISTO_NUMBER_OF_LEVELS_ERROR: + return "NPP_HISTOGRAM_NUMBER_OF_LEVELS_ERROR"; + + case NPP_INVALID_INPUT: + return "NPP_INVALID_INPUT"; + + case NPP_POINTER_ERROR: + return "NPP_POINTER_ERROR"; + + case NPP_WARNING: + return "NPP_WARNING"; + + case NPP_ODD_ROI_WARNING: + return "NPP_ODD_ROI_WARNING"; +#else + + // These are for CUDA 5.5 or higher + case NPP_BAD_ARGUMENT_ERROR: + return "NPP_BAD_ARGUMENT_ERROR"; + + case NPP_COEFFICIENT_ERROR: + return "NPP_COEFFICIENT_ERROR"; + + case NPP_RECTANGLE_ERROR: + return "NPP_RECTANGLE_ERROR"; + + case NPP_QUADRANGLE_ERROR: + return "NPP_QUADRANGLE_ERROR"; + + case NPP_MEMORY_ALLOCATION_ERR: + return "NPP_MEMORY_ALLOCATION_ERROR"; + + case NPP_HISTOGRAM_NUMBER_OF_LEVELS_ERROR: + return "NPP_HISTOGRAM_NUMBER_OF_LEVELS_ERROR"; + + case NPP_INVALID_HOST_POINTER_ERROR: + return "NPP_INVALID_HOST_POINTER_ERROR"; + + case NPP_INVALID_DEVICE_POINTER_ERROR: + return "NPP_INVALID_DEVICE_POINTER_ERROR"; +#endif + + case NPP_LUT_NUMBER_OF_LEVELS_ERROR: + return "NPP_LUT_NUMBER_OF_LEVELS_ERROR"; + + case NPP_TEXTURE_BIND_ERROR: + return "NPP_TEXTURE_BIND_ERROR"; + + case NPP_WRONG_INTERSECTION_ROI_ERROR: + return "NPP_WRONG_INTERSECTION_ROI_ERROR"; + + case NPP_NOT_EVEN_STEP_ERROR: + return "NPP_NOT_EVEN_STEP_ERROR"; + + case NPP_INTERPOLATION_ERROR: + return "NPP_INTERPOLATION_ERROR"; + + case NPP_RESIZE_FACTOR_ERROR: + return "NPP_RESIZE_FACTOR_ERROR"; + + case NPP_HAAR_CLASSIFIER_PIXEL_MATCH_ERROR: + return "NPP_HAAR_CLASSIFIER_PIXEL_MATCH_ERROR"; + +#if ((NPP_VERSION_MAJOR << 12) + (NPP_VERSION_MINOR << 4)) <= 0x5000 + + case NPP_MEMFREE_ERR: + return "NPP_MEMFREE_ERR"; + + case NPP_MEMSET_ERR: + return "NPP_MEMSET_ERR"; + + case NPP_MEMCPY_ERR: + return "NPP_MEMCPY_ERROR"; + + case NPP_MIRROR_FLIP_ERR: + return "NPP_MIRROR_FLIP_ERR"; +#else + + case NPP_MEMFREE_ERROR: + return "NPP_MEMFREE_ERROR"; + + case NPP_MEMSET_ERROR: + return "NPP_MEMSET_ERROR"; + + case NPP_MEMCPY_ERROR: + return "NPP_MEMCPY_ERROR"; + + case NPP_MIRROR_FLIP_ERROR: + return "NPP_MIRROR_FLIP_ERROR"; +#endif + + case NPP_ALIGNMENT_ERROR: + return "NPP_ALIGNMENT_ERROR"; + + case NPP_STEP_ERROR: + return "NPP_STEP_ERROR"; + + case NPP_SIZE_ERROR: + return "NPP_SIZE_ERROR"; + + case NPP_NULL_POINTER_ERROR: + return "NPP_NULL_POINTER_ERROR"; + + case NPP_CUDA_KERNEL_EXECUTION_ERROR: + return "NPP_CUDA_KERNEL_EXECUTION_ERROR"; + + case NPP_NOT_IMPLEMENTED_ERROR: + return "NPP_NOT_IMPLEMENTED_ERROR"; + + case NPP_ERROR: + return "NPP_ERROR"; + + case NPP_SUCCESS: + return "NPP_SUCCESS"; + + case NPP_WRONG_INTERSECTION_QUAD_WARNING: + return "NPP_WRONG_INTERSECTION_QUAD_WARNING"; + + case NPP_MISALIGNED_DST_ROI_WARNING: + return "NPP_MISALIGNED_DST_ROI_WARNING"; + + case NPP_AFFINE_QUAD_INCORRECT_WARNING: + return "NPP_AFFINE_QUAD_INCORRECT_WARNING"; + + case NPP_DOUBLE_SIZE_WARNING: + return "NPP_DOUBLE_SIZE_WARNING"; + + case NPP_WRONG_INTERSECTION_ROI_WARNING: + return "NPP_WRONG_INTERSECTION_ROI_WARNING"; + +#if ((NPP_VERSION_MAJOR << 12) + (NPP_VERSION_MINOR << 4)) >= 0x6000 + /* These are 6.0 or higher */ + case NPP_LUT_PALETTE_BITSIZE_ERROR: + return "NPP_LUT_PALETTE_BITSIZE_ERROR"; + + case NPP_ZC_MODE_NOT_SUPPORTED_ERROR: + return "NPP_ZC_MODE_NOT_SUPPORTED_ERROR"; + + case NPP_QUALITY_INDEX_ERROR: + return "NPP_QUALITY_INDEX_ERROR"; + + case NPP_CHANNEL_ORDER_ERROR: + return "NPP_CHANNEL_ORDER_ERROR"; + + case NPP_ZERO_MASK_VALUE_ERROR: + return "NPP_ZERO_MASK_VALUE_ERROR"; + + case NPP_NUMBER_OF_CHANNELS_ERROR: + return "NPP_NUMBER_OF_CHANNELS_ERROR"; + + case NPP_COI_ERROR: + return "NPP_COI_ERROR"; + + case NPP_DIVISOR_ERROR: + return "NPP_DIVISOR_ERROR"; + + case NPP_CHANNEL_ERROR: + return "NPP_CHANNEL_ERROR"; + + case NPP_STRIDE_ERROR: + return "NPP_STRIDE_ERROR"; + + case NPP_ANCHOR_ERROR: + return "NPP_ANCHOR_ERROR"; + + case NPP_MASK_SIZE_ERROR: + return "NPP_MASK_SIZE_ERROR"; + + case NPP_MOMENT_00_ZERO_ERROR: + return "NPP_MOMENT_00_ZERO_ERROR"; + + case NPP_THRESHOLD_NEGATIVE_LEVEL_ERROR: + return "NPP_THRESHOLD_NEGATIVE_LEVEL_ERROR"; + + case NPP_THRESHOLD_ERROR: + return "NPP_THRESHOLD_ERROR"; + + case NPP_CONTEXT_MATCH_ERROR: + return "NPP_CONTEXT_MATCH_ERROR"; + + case NPP_FFT_FLAG_ERROR: + return "NPP_FFT_FLAG_ERROR"; + + case NPP_FFT_ORDER_ERROR: + return "NPP_FFT_ORDER_ERROR"; + + case NPP_SCALE_RANGE_ERROR: + return "NPP_SCALE_RANGE_ERROR"; + + case NPP_DATA_TYPE_ERROR: + return "NPP_DATA_TYPE_ERROR"; + + case NPP_OUT_OFF_RANGE_ERROR: + return "NPP_OUT_OFF_RANGE_ERROR"; + + case NPP_DIVIDE_BY_ZERO_ERROR: + return "NPP_DIVIDE_BY_ZERO_ERROR"; + + case NPP_RANGE_ERROR: + return "NPP_RANGE_ERROR"; + + case NPP_NO_MEMORY_ERROR: + return "NPP_NO_MEMORY_ERROR"; + + case NPP_ERROR_RESERVED: + return "NPP_ERROR_RESERVED"; + + case NPP_NO_OPERATION_WARNING: + return "NPP_NO_OPERATION_WARNING"; + + case NPP_DIVIDE_BY_ZERO_WARNING: + return "NPP_DIVIDE_BY_ZERO_WARNING"; +#endif + +#if ((NPP_VERSION_MAJOR << 12) + (NPP_VERSION_MINOR << 4)) >= 0x7000 + /* These are 7.0 or higher */ + case NPP_OVERFLOW_ERROR: + return "NPP_OVERFLOW_ERROR"; + + case NPP_CORRUPTED_DATA_ERROR: + return "NPP_CORRUPTED_DATA_ERROR"; +#endif + } + + return ""; +} +#endif + +template +void check(T result, char const *const func, const char *const file, + int const line) { + if (result == cudaErrorCallRequiresNewerDriver) { + return; + } + if (result) { + fprintf(stderr, "CUDA error at %s:%d code=%d(%s) \"%s\" \n", file, line, + static_cast(result), _cudaGetErrorEnum(result), func); + exit(EXIT_FAILURE); + } +} + +#ifdef __DRIVER_TYPES_H__ +// This will output the proper CUDA error strings in the event +// that a CUDA host call returns an error +#define checkCudaErrors(val) check((val), #val, __FILE__, __LINE__) + +// This will output the proper error string when calling cudaGetLastError +#define getLastCudaError(msg) __getLastCudaError(msg, __FILE__, __LINE__) + +inline void __getLastCudaError(const char *errorMessage, const char *file, + const int line) { + cudaError_t err = cudaGetLastError(); + + if (cudaSuccess != err) { + fprintf(stderr, + "%s(%i) : getLastCudaError() CUDA error :" + " %s : (%d) %s.\n", + file, line, errorMessage, static_cast(err), + cudaGetErrorString(err)); + exit(EXIT_FAILURE); + } +} + +// This will only print the proper error string when calling cudaGetLastError +// but not exit program incase error detected. +#define printLastCudaError(msg) __printLastCudaError(msg, __FILE__, __LINE__) + +inline void __printLastCudaError(const char *errorMessage, const char *file, + const int line) { + cudaError_t err = cudaGetLastError(); + + if (cudaSuccess != err) { + fprintf(stderr, + "%s(%i) : getLastCudaError() CUDA error :" + " %s : (%d) %s.\n", + file, line, errorMessage, static_cast(err), + cudaGetErrorString(err)); + } +} +#endif + +#ifndef MAX +#define MAX(a, b) (a > b ? a : b) +#endif + +// Float To Int conversion +inline int ftoi(float value) { + return (value >= 0 ? static_cast(value + 0.5) + : static_cast(value - 0.5)); +} + +// Beginning of GPU Architecture definitions +inline int _ConvertSMVer2Cores(int major, int minor) { + // Defines for GPU Architecture types (using the SM version to determine + // the # of cores per SM + typedef struct { + int SM; // 0xMm (hexidecimal notation), M = SM Major version, + // and m = SM minor version + int Cores; + } sSMtoCores; + + sSMtoCores nGpuArchCoresPerSM[] = { + {0x30, 192}, + {0x32, 192}, + {0x35, 192}, + {0x37, 192}, + {0x50, 128}, + {0x52, 128}, + {0x53, 128}, + {0x60, 64}, + {0x61, 128}, + {0x62, 128}, + {0x70, 64}, + {0x72, 64}, + {0x75, 64}, + {0x80, 64}, + {0x86, 128}, + {0x87, 128}, + {0x89, 128}, + {0x90, 128}, + {-1, -1}}; + + int index = 0; + + while (nGpuArchCoresPerSM[index].SM != -1) { + if (nGpuArchCoresPerSM[index].SM == ((major << 4) + minor)) { + return nGpuArchCoresPerSM[index].Cores; + } + + index++; + } + + // If we don't find the values, we default use the previous one + // to run properly + printf( + "MapSMtoCores for SM %d.%d is undefined." + " Default to use %d Cores/SM\n", + major, minor, nGpuArchCoresPerSM[index - 1].Cores); + return nGpuArchCoresPerSM[index - 1].Cores; +} + +inline const char* _ConvertSMVer2ArchName(int major, int minor) { + // Defines for GPU Architecture types (using the SM version to determine + // the GPU Arch name) + typedef struct { + int SM; // 0xMm (hexidecimal notation), M = SM Major version, + // and m = SM minor version + const char* name; + } sSMtoArchName; + + sSMtoArchName nGpuArchNameSM[] = { + {0x30, "Kepler"}, + {0x32, "Kepler"}, + {0x35, "Kepler"}, + {0x37, "Kepler"}, + {0x50, "Maxwell"}, + {0x52, "Maxwell"}, + {0x53, "Maxwell"}, + {0x60, "Pascal"}, + {0x61, "Pascal"}, + {0x62, "Pascal"}, + {0x70, "Volta"}, + {0x72, "Xavier"}, + {0x75, "Turing"}, + {0x80, "Ampere"}, + {0x86, "Ampere"}, + {0x87, "Ampere"}, + {0x89, "Ada"}, + {0x90, "Hopper"}, + {-1, "Graphics Device"}}; + + int index = 0; + + while (nGpuArchNameSM[index].SM != -1) { + if (nGpuArchNameSM[index].SM == ((major << 4) + minor)) { + return nGpuArchNameSM[index].name; + } + + index++; + } + + // If we don't find the values, we default use the previous one + // to run properly + printf( + "MapSMtoArchName for SM %d.%d is undefined." + " Default to use %s\n", + major, minor, nGpuArchNameSM[index - 1].name); + return nGpuArchNameSM[index - 1].name; +} + // end of GPU Architecture definitions + +#ifdef __CUDA_RUNTIME_H__ +// General GPU Device CUDA Initialization +inline int gpuDeviceInit(int devID) { + int device_count; + checkCudaErrors(cudaGetDeviceCount(&device_count)); + + if (device_count == 0) { + fprintf(stderr, + "gpuDeviceInit() CUDA error: " + "no devices supporting CUDA.\n"); + exit(EXIT_FAILURE); + } + + if (devID < 0) { + devID = 0; + } + + if (devID > device_count - 1) { + fprintf(stderr, "\n"); + fprintf(stderr, ">> %d CUDA capable GPU device(s) detected. <<\n", + device_count); + fprintf(stderr, + ">> gpuDeviceInit (-device=%d) is not a valid" + " GPU device. <<\n", + devID); + fprintf(stderr, "\n"); + return -devID; + } + + int computeMode = -1, major = 0, minor = 0; + checkCudaErrors(cudaDeviceGetAttribute(&computeMode, cudaDevAttrComputeMode, devID)); + checkCudaErrors(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, devID)); + checkCudaErrors(cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, devID)); + if (computeMode == cudaComputeModeProhibited) { + fprintf(stderr, + "Error: device is running in , no threads can use cudaSetDevice().\n"); + return -1; + } + + if (major < 1) { + fprintf(stderr, "gpuDeviceInit(): GPU device does not support CUDA.\n"); + exit(EXIT_FAILURE); + } + + checkCudaErrors(cudaSetDevice(devID)); + printf("gpuDeviceInit() CUDA Device [%d]: \"%s\n", devID, _ConvertSMVer2ArchName(major, minor)); + + return devID; +} + +// This function returns the best GPU (with maximum GFLOPS) +inline int gpuGetMaxGflopsDeviceId() { + int current_device = 0, sm_per_multiproc = 0; + int max_perf_device = 0; + int device_count = 0; + int devices_prohibited = 0; + + uint64_t max_compute_perf = 0; + checkCudaErrors(cudaGetDeviceCount(&device_count)); + + if (device_count == 0) { + fprintf(stderr, + "gpuGetMaxGflopsDeviceId() CUDA error:" + " no devices supporting CUDA.\n"); + exit(EXIT_FAILURE); + } + + // Find the best CUDA capable GPU device + current_device = 0; + + while (current_device < device_count) { + int computeMode = -1, major = 0, minor = 0; + checkCudaErrors(cudaDeviceGetAttribute(&computeMode, cudaDevAttrComputeMode, current_device)); + checkCudaErrors(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, current_device)); + checkCudaErrors(cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, current_device)); + + // If this GPU is not running on Compute Mode prohibited, + // then we can add it to the list + if (computeMode != cudaComputeModeProhibited) { + if (major == 9999 && minor == 9999) { + sm_per_multiproc = 1; + } else { + sm_per_multiproc = + _ConvertSMVer2Cores(major, minor); + } + int multiProcessorCount = 0, clockRate = 0; + checkCudaErrors(cudaDeviceGetAttribute(&multiProcessorCount, cudaDevAttrMultiProcessorCount, current_device)); + cudaError_t result = cudaDeviceGetAttribute(&clockRate, cudaDevAttrClockRate, current_device); + if (result != cudaSuccess) { + // If cudaDevAttrClockRate attribute is not supported we + // set clockRate as 1, to consider GPU with most SMs and CUDA Cores. + if(result == cudaErrorInvalidValue) { + clockRate = 1; + } + else { + fprintf(stderr, "CUDA error at %s:%d code=%d(%s) \n", __FILE__, __LINE__, + static_cast(result), _cudaGetErrorEnum(result)); + exit(EXIT_FAILURE); + } + } + uint64_t compute_perf = (uint64_t)multiProcessorCount * sm_per_multiproc * clockRate; + + if (compute_perf > max_compute_perf) { + max_compute_perf = compute_perf; + max_perf_device = current_device; + } + } else { + devices_prohibited++; + } + + ++current_device; + } + + if (devices_prohibited == device_count) { + fprintf(stderr, + "gpuGetMaxGflopsDeviceId() CUDA error:" + " all devices have compute mode prohibited.\n"); + exit(EXIT_FAILURE); + } + + return max_perf_device; +} + +// Initialization code to find the best CUDA Device +inline int findCudaDevice(int argc, const char **argv) { + int devID = 0; + + // If the command-line has a device number specified, use it + if (checkCmdLineFlag(argc, argv, "device")) { + devID = getCmdLineArgumentInt(argc, argv, "device="); + + if (devID < 0) { + printf("Invalid command line parameter\n "); + exit(EXIT_FAILURE); + } else { + devID = gpuDeviceInit(devID); + + if (devID < 0) { + printf("exiting...\n"); + exit(EXIT_FAILURE); + } + } + } else { + // Otherwise pick the device with highest Gflops/s + devID = gpuGetMaxGflopsDeviceId(); + checkCudaErrors(cudaSetDevice(devID)); + int major = 0, minor = 0; + checkCudaErrors(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, devID)); + checkCudaErrors(cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, devID)); + printf("GPU Device %d: \"%s\" with compute capability %d.%d\n\n", + devID, _ConvertSMVer2ArchName(major, minor), major, minor); + + } + + return devID; +} + +inline int findIntegratedGPU() { + int current_device = 0; + int device_count = 0; + int devices_prohibited = 0; + + checkCudaErrors(cudaGetDeviceCount(&device_count)); + + if (device_count == 0) { + fprintf(stderr, "CUDA error: no devices supporting CUDA.\n"); + exit(EXIT_FAILURE); + } + + // Find the integrated GPU which is compute capable + while (current_device < device_count) { + int computeMode = -1, integrated = -1; + checkCudaErrors(cudaDeviceGetAttribute(&computeMode, cudaDevAttrComputeMode, current_device)); + checkCudaErrors(cudaDeviceGetAttribute(&integrated, cudaDevAttrIntegrated, current_device)); + // If GPU is integrated and is not running on Compute Mode prohibited, + // then cuda can map to GLES resource + if (integrated && (computeMode != cudaComputeModeProhibited)) { + checkCudaErrors(cudaSetDevice(current_device)); + + int major = 0, minor = 0; + checkCudaErrors(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, current_device)); + checkCudaErrors(cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, current_device)); + printf("GPU Device %d: \"%s\" with compute capability %d.%d\n\n", + current_device, _ConvertSMVer2ArchName(major, minor), major, minor); + + return current_device; + } else { + devices_prohibited++; + } + + current_device++; + } + + if (devices_prohibited == device_count) { + fprintf(stderr, + "CUDA error:" + " No GLES-CUDA Interop capable GPU found.\n"); + exit(EXIT_FAILURE); + } + + return -1; +} + +// General check for CUDA GPU SM Capabilities +inline bool checkCudaCapabilities(int major_version, int minor_version) { + int dev; + int major = 0, minor = 0; + + checkCudaErrors(cudaGetDevice(&dev)); + checkCudaErrors(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev)); + checkCudaErrors(cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev)); + + if ((major > major_version) || + (major == major_version && + minor >= minor_version)) { + printf(" Device %d: <%16s >, Compute SM %d.%d detected\n", dev, + _ConvertSMVer2ArchName(major, minor), major, minor); + return true; + } else { + printf( + " No GPU device was found that can support " + "CUDA compute capability %d.%d.\n", + major_version, minor_version); + return false; + } +} +#endif + + // end of CUDA Helper Functions + +#endif // COMMON_HELPER_CUDA_H_ diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_functions.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_functions.h new file mode 100644 index 0000000..2975ddb --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_functions.h @@ -0,0 +1,59 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// These are helper functions for the SDK samples (string parsing, +// timers, image helpers, etc) +#ifndef COMMON_HELPER_FUNCTIONS_H_ +#define COMMON_HELPER_FUNCTIONS_H_ + +#ifdef WIN32 +#pragma warning(disable : 4996) +#endif + +// includes, project +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +// includes, timer, string parsing, image helpers +#include // helper functions for image compare, dump, data comparisons +#include // helper functions for string parsing +#include // helper functions for timers + +#ifndef EXIT_WAIVED +#define EXIT_WAIVED 2 +#endif + +#endif // COMMON_HELPER_FUNCTIONS_H_ diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_image.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_image.h new file mode 100644 index 0000000..9b7edc0 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_image.h @@ -0,0 +1,1001 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// These are helper functions for the SDK samples (image,bitmap) +#ifndef COMMON_HELPER_IMAGE_H_ +#define COMMON_HELPER_IMAGE_H_ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifndef MIN +#define MIN(a, b) ((a < b) ? a : b) +#endif +#ifndef MAX +#define MAX(a, b) ((a > b) ? a : b) +#endif + +#ifndef EXIT_WAIVED +#define EXIT_WAIVED 2 +#endif + +#include + +// namespace unnamed (internal) +namespace helper_image_internal { +//! size of PGM file header +const unsigned int PGMHeaderSize = 0x40; + +// types + +//! Data converter from unsigned char / unsigned byte to type T +template +struct ConverterFromUByte; + +//! Data converter from unsigned char / unsigned byte +template <> +struct ConverterFromUByte { + //! Conversion operator + //! @return converted value + //! @param val value to convert + float operator()(const unsigned char &val) { + return static_cast(val); + } +}; + +//! Data converter from unsigned char / unsigned byte to float +template <> +struct ConverterFromUByte { + //! Conversion operator + //! @return converted value + //! @param val value to convert + float operator()(const unsigned char &val) { + return static_cast(val) / 255.0f; + } +}; + +//! Data converter from unsigned char / unsigned byte to type T +template +struct ConverterToUByte; + +//! Data converter from unsigned char / unsigned byte to unsigned int +template <> +struct ConverterToUByte { + //! Conversion operator (essentially a passthru + //! @return converted value + //! @param val value to convert + unsigned char operator()(const unsigned char &val) { return val; } +}; + +//! Data converter from unsigned char / unsigned byte to unsigned int +template <> +struct ConverterToUByte { + //! Conversion operator + //! @return converted value + //! @param val value to convert + unsigned char operator()(const float &val) { + return static_cast(val * 255.0f); + } +}; +} // namespace helper_image_internal + +#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) +#ifndef FOPEN +#define FOPEN(fHandle, filename, mode) fopen_s(&fHandle, filename, mode) +#endif +#ifndef FOPEN_FAIL +#define FOPEN_FAIL(result) (result != 0) +#endif +#ifndef SSCANF +#define SSCANF sscanf_s +#endif +#else +#ifndef FOPEN +#define FOPEN(fHandle, filename, mode) (fHandle = fopen(filename, mode)) +#endif +#ifndef FOPEN_FAIL +#define FOPEN_FAIL(result) (result == NULL) +#endif +#ifndef SSCANF +#define SSCANF sscanf +#endif +#endif + +inline bool __loadPPM(const char *file, unsigned char **data, unsigned int *w, + unsigned int *h, unsigned int *channels) { + FILE *fp = NULL; + + if (FOPEN_FAIL(FOPEN(fp, file, "rb"))) { + std::cerr << "__LoadPPM() : Failed to open file: " << file << std::endl; + return false; + } + + // check header + char header[helper_image_internal::PGMHeaderSize]; + + if (fgets(header, helper_image_internal::PGMHeaderSize, fp) == NULL) { + std::cerr << "__LoadPPM() : reading PGM header returned NULL" << std::endl; + return false; + } + + if (strncmp(header, "P5", 2) == 0) { + *channels = 1; + } else if (strncmp(header, "P6", 2) == 0) { + *channels = 3; + } else { + std::cerr << "__LoadPPM() : File is not a PPM or PGM image" << std::endl; + *channels = 0; + return false; + } + + // parse header, read maxval, width and height + unsigned int width = 0; + unsigned int height = 0; + unsigned int maxval = 0; + unsigned int i = 0; + + while (i < 3) { + if (fgets(header, helper_image_internal::PGMHeaderSize, fp) == NULL) { + std::cerr << "__LoadPPM() : reading PGM header returned NULL" + << std::endl; + return false; + } + + if (header[0] == '#') { + continue; + } + + if (i == 0) { + i += SSCANF(header, "%u %u %u", &width, &height, &maxval); + } else if (i == 1) { + i += SSCANF(header, "%u %u", &height, &maxval); + } else if (i == 2) { + i += SSCANF(header, "%u", &maxval); + } + } + + // check if given handle for the data is initialized + if (NULL != *data) { + if (*w != width || *h != height) { + std::cerr << "__LoadPPM() : Invalid image dimensions." << std::endl; + } + } else { + *data = (unsigned char *)malloc(sizeof(unsigned char) * width * height * + *channels); + *w = width; + *h = height; + } + + // read and close file + if (fread(*data, sizeof(unsigned char), width * height * *channels, fp) == + 0) { + std::cerr << "__LoadPPM() read data returned error." << std::endl; + } + + fclose(fp); + + return true; +} + +template +inline bool sdkLoadPGM(const char *file, T **data, unsigned int *w, + unsigned int *h) { + unsigned char *idata = NULL; + unsigned int channels; + + if (true != __loadPPM(file, &idata, w, h, &channels)) { + return false; + } + + unsigned int size = *w * *h * channels; + + // initialize mem if necessary + // the correct size is checked / set in loadPGMc() + if (NULL == *data) { + *data = reinterpret_cast(malloc(sizeof(T) * size)); + } + + // copy and cast data + std::transform(idata, idata + size, *data, + helper_image_internal::ConverterFromUByte()); + + free(idata); + + return true; +} + +template +inline bool sdkLoadPPM4(const char *file, T **data, unsigned int *w, + unsigned int *h) { + unsigned char *idata = 0; + unsigned int channels; + + if (__loadPPM(file, &idata, w, h, &channels)) { + // pad 4th component + int size = *w * *h; + // keep the original pointer + unsigned char *idata_orig = idata; + *data = reinterpret_cast(malloc(sizeof(T) * size * 4)); + unsigned char *ptr = *data; + + for (int i = 0; i < size; i++) { + *ptr++ = *idata++; + *ptr++ = *idata++; + *ptr++ = *idata++; + *ptr++ = 0; + } + + free(idata_orig); + return true; + } else { + free(idata); + return false; + } +} + +inline bool __savePPM(const char *file, unsigned char *data, unsigned int w, + unsigned int h, unsigned int channels) { + assert(NULL != data); + assert(w > 0); + assert(h > 0); + + std::fstream fh(file, std::fstream::out | std::fstream::binary); + + if (fh.bad()) { + std::cerr << "__savePPM() : Opening file failed." << std::endl; + return false; + } + + if (channels == 1) { + fh << "P5\n"; + } else if (channels == 3) { + fh << "P6\n"; + } else { + std::cerr << "__savePPM() : Invalid number of channels." << std::endl; + return false; + } + + fh << w << "\n" << h << "\n" << 0xff << std::endl; + + for (unsigned int i = 0; (i < (w * h * channels)) && fh.good(); ++i) { + fh << data[i]; + } + + fh.flush(); + + if (fh.bad()) { + std::cerr << "__savePPM() : Writing data failed." << std::endl; + return false; + } + + fh.close(); + + return true; +} + +template +inline bool sdkSavePGM(const char *file, T *data, unsigned int w, + unsigned int h) { + unsigned int size = w * h; + unsigned char *idata = (unsigned char *)malloc(sizeof(unsigned char) * size); + + std::transform(data, data + size, idata, + helper_image_internal::ConverterToUByte()); + + // write file + bool result = __savePPM(file, idata, w, h, 1); + + // cleanup + free(idata); + + return result; +} + +inline bool sdkSavePPM4ub(const char *file, unsigned char *data, unsigned int w, + unsigned int h) { + // strip 4th component + int size = w * h; + unsigned char *ndata = + (unsigned char *)malloc(sizeof(unsigned char) * size * 3); + unsigned char *ptr = ndata; + + for (int i = 0; i < size; i++) { + *ptr++ = *data++; + *ptr++ = *data++; + *ptr++ = *data++; + data++; + } + + bool result = __savePPM(file, ndata, w, h, 3); + free(ndata); + return result; +} + +////////////////////////////////////////////////////////////////////////////// +//! Read file \filename and return the data +//! @return bool if reading the file succeeded, otherwise false +//! @param filename name of the source file +//! @param data uninitialized pointer, returned initialized and pointing to +//! the data read +//! @param len number of data elements in data, -1 on error +////////////////////////////////////////////////////////////////////////////// +template +inline bool sdkReadFile(const char *filename, T **data, unsigned int *len, + bool verbose) { + // check input arguments + assert(NULL != filename); + assert(NULL != len); + + // intermediate storage for the data read + std::vector data_read; + + // open file for reading + FILE *fh = NULL; + + // check if filestream is valid + if (FOPEN_FAIL(FOPEN(fh, filename, "r"))) { + printf("Unable to open input file: %s\n", filename); + return false; + } + + // read all data elements + T token; + + while (!feof(fh)) { + fscanf(fh, "%f", &token); + data_read.push_back(token); + } + + // the last element is read twice + data_read.pop_back(); + fclose(fh); + + // check if the given handle is already initialized + if (NULL != *data) { + if (*len != data_read.size()) { + std::cerr << "sdkReadFile() : Initialized memory given but " + << "size mismatch with signal read " + << "(data read / data init = " << (unsigned int)data_read.size() + << " / " << *len << ")" << std::endl; + + return false; + } + } else { + // allocate storage for the data read + *data = reinterpret_cast(malloc(sizeof(T) * data_read.size())); + // store signal size + *len = static_cast(data_read.size()); + } + + // copy data + memcpy(*data, &data_read.front(), sizeof(T) * data_read.size()); + + return true; +} + +////////////////////////////////////////////////////////////////////////////// +//! Read file \filename and return the data +//! @return bool if reading the file succeeded, otherwise false +//! @param filename name of the source file +//! @param data uninitialized pointer, returned initialized and pointing to +//! the data read +//! @param len number of data elements in data, -1 on error +////////////////////////////////////////////////////////////////////////////// +template +inline bool sdkReadFileBlocks(const char *filename, T **data, unsigned int *len, + unsigned int block_num, unsigned int block_size, + bool verbose) { + // check input arguments + assert(NULL != filename); + assert(NULL != len); + + // open file for reading + FILE *fh = fopen(filename, "rb"); + + if (fh == NULL && verbose) { + std::cerr << "sdkReadFile() : Opening file failed." << std::endl; + return false; + } + + // check if the given handle is already initialized + // allocate storage for the data read + data[block_num] = reinterpret_cast(malloc(block_size)); + + // read all data elements + fseek(fh, block_num * block_size, SEEK_SET); + *len = fread(data[block_num], sizeof(T), block_size / sizeof(T), fh); + + fclose(fh); + + return true; +} + +////////////////////////////////////////////////////////////////////////////// +//! Write a data file \filename +//! @return true if writing the file succeeded, otherwise false +//! @param filename name of the source file +//! @param data data to write +//! @param len number of data elements in data, -1 on error +//! @param epsilon epsilon for comparison +////////////////////////////////////////////////////////////////////////////// +template +inline bool sdkWriteFile(const char *filename, const T *data, unsigned int len, + const S epsilon, bool verbose, bool append = false) { + assert(NULL != filename); + assert(NULL != data); + + // open file for writing + // if (append) { + std::fstream fh(filename, std::fstream::out | std::fstream::ate); + + if (verbose) { + std::cerr << "sdkWriteFile() : Open file " << filename + << " for write/append." << std::endl; + } + + /* } else { + std::fstream fh(filename, std::fstream::out); + if (verbose) { + std::cerr << "sdkWriteFile() : Open file " << filename << " for + write." << std::endl; + } + } + */ + + // check if filestream is valid + if (!fh.good()) { + if (verbose) { + std::cerr << "sdkWriteFile() : Opening file failed." << std::endl; + } + + return false; + } + + // first write epsilon + fh << "# " << epsilon << "\n"; + + // write data + for (unsigned int i = 0; (i < len) && (fh.good()); ++i) { + fh << data[i] << ' '; + } + + // Check if writing succeeded + if (!fh.good()) { + if (verbose) { + std::cerr << "sdkWriteFile() : Writing file failed." << std::endl; + } + + return false; + } + + // file ends with nl + fh << std::endl; + + return true; +} + +////////////////////////////////////////////////////////////////////////////// +//! Compare two arrays of arbitrary type +//! @return true if \a reference and \a data are identical, otherwise false +//! @param reference timer_interface to the reference data / gold image +//! @param data handle to the computed data +//! @param len number of elements in reference and data +//! @param epsilon epsilon to use for the comparison +////////////////////////////////////////////////////////////////////////////// +template +inline bool compareData(const T *reference, const T *data, + const unsigned int len, const S epsilon, + const float threshold) { + assert(epsilon >= 0); + + bool result = true; + unsigned int error_count = 0; + + for (unsigned int i = 0; i < len; ++i) { + float diff = static_cast(reference[i]) - static_cast(data[i]); + bool comp = (diff <= epsilon) && (diff >= -epsilon); + result &= comp; + + error_count += !comp; + +#if 0 + + if (!comp) { + std::cerr << "ERROR, i = " << i << ",\t " + << reference[i] << " / " + << data[i] + << " (reference / data)\n"; + } + +#endif + } + + if (threshold == 0.0f) { + return (result) ? true : false; + } else { + if (error_count) { + printf("%4.2f(%%) of bytes mismatched (count=%d)\n", + static_cast(error_count) * 100 / static_cast(len), + error_count); + } + + return (len * threshold > error_count) ? true : false; + } +} + +#ifndef __MIN_EPSILON_ERROR +#define __MIN_EPSILON_ERROR 1e-3f +#endif + +////////////////////////////////////////////////////////////////////////////// +//! Compare two arrays of arbitrary type +//! @return true if \a reference and \a data are identical, otherwise false +//! @param reference handle to the reference data / gold image +//! @param data handle to the computed data +//! @param len number of elements in reference and data +//! @param epsilon epsilon to use for the comparison +//! @param epsilon threshold % of (# of bytes) for pass/fail +////////////////////////////////////////////////////////////////////////////// +template +inline bool compareDataAsFloatThreshold(const T *reference, const T *data, + const unsigned int len, const S epsilon, + const float threshold) { + assert(epsilon >= 0); + + // If we set epsilon to be 0, let's set a minimum threshold + float max_error = MAX((float)epsilon, __MIN_EPSILON_ERROR); + int error_count = 0; + bool result = true; + + for (unsigned int i = 0; i < len; ++i) { + float diff = + fabs(static_cast(reference[i]) - static_cast(data[i])); + bool comp = (diff < max_error); + result &= comp; + + if (!comp) { + error_count++; + } + } + + if (threshold == 0.0f) { + if (error_count) { + printf("total # of errors = %d\n", error_count); + } + + return (error_count == 0) ? true : false; + } else { + if (error_count) { + printf("%4.2f(%%) of bytes mismatched (count=%d)\n", + static_cast(error_count) * 100 / static_cast(len), + error_count); + } + + return ((len * threshold > error_count) ? true : false); + } +} + +inline void sdkDumpBin(void *data, unsigned int bytes, const char *filename) { + printf("sdkDumpBin: <%s>\n", filename); + FILE *fp; + FOPEN(fp, filename, "wb"); + fwrite(data, bytes, 1, fp); + fflush(fp); + fclose(fp); +} + +inline bool sdkCompareBin2BinUint(const char *src_file, const char *ref_file, + unsigned int nelements, const float epsilon, + const float threshold, char *exec_path) { + unsigned int *src_buffer, *ref_buffer; + FILE *src_fp = NULL, *ref_fp = NULL; + + uint64_t error_count = 0; + size_t fsize = 0; + + if (FOPEN_FAIL(FOPEN(src_fp, src_file, "rb"))) { + printf("compareBin2Bin unable to open src_file: %s\n", + src_file); + error_count++; + } + + char *ref_file_path = sdkFindFilePath(ref_file, exec_path); + + if (ref_file_path == NULL) { + printf("compareBin2Bin unable to find <%s> in <%s>\n", + ref_file, exec_path); + printf(">>> Check info.xml and [project//data] folder <%s> <<<\n", + ref_file); + printf("Aborting comparison!\n"); + printf(" FAILED\n"); + error_count++; + + if (src_fp) { + fclose(src_fp); + } + + if (ref_fp) { + fclose(ref_fp); + } + } else { + if (FOPEN_FAIL(FOPEN(ref_fp, ref_file_path, "rb"))) { + printf( + "compareBin2Bin " + " unable to open ref_file: %s\n", + ref_file_path); + error_count++; + } + + if (src_fp && ref_fp) { + src_buffer = (unsigned int *)malloc(nelements * sizeof(unsigned int)); + ref_buffer = (unsigned int *)malloc(nelements * sizeof(unsigned int)); + + fsize = fread(src_buffer, nelements, sizeof(unsigned int), src_fp); + fsize = fread(ref_buffer, nelements, sizeof(unsigned int), ref_fp); + + printf( + "> compareBin2Bin nelements=%d," + " epsilon=%4.2f, threshold=%4.2f\n", + nelements, epsilon, threshold); + printf(" src_file <%s>, size=%d bytes\n", src_file, + static_cast(fsize)); + printf(" ref_file <%s>, size=%d bytes\n", ref_file_path, + static_cast(fsize)); + + if (!compareData(ref_buffer, src_buffer, nelements, + epsilon, threshold)) { + error_count++; + } + + fclose(src_fp); + fclose(ref_fp); + + free(src_buffer); + free(ref_buffer); + } else { + if (src_fp) { + fclose(src_fp); + } + + if (ref_fp) { + fclose(ref_fp); + } + } + } + + if (error_count == 0) { + printf(" OK\n"); + } else { + printf(" FAILURE: %d errors...\n", (unsigned int)error_count); + } + + return (error_count == 0); // returns true if all pixels pass +} + +inline bool sdkCompareBin2BinFloat(const char *src_file, const char *ref_file, + unsigned int nelements, const float epsilon, + const float threshold, char *exec_path) { + float *src_buffer = NULL, *ref_buffer = NULL; + FILE *src_fp = NULL, *ref_fp = NULL; + size_t fsize = 0; + + uint64_t error_count = 0; + + if (FOPEN_FAIL(FOPEN(src_fp, src_file, "rb"))) { + printf("compareBin2Bin unable to open src_file: %s\n", src_file); + error_count = 1; + } + + char *ref_file_path = sdkFindFilePath(ref_file, exec_path); + + if (ref_file_path == NULL) { + printf("compareBin2Bin unable to find <%s> in <%s>\n", ref_file, + exec_path); + printf(">>> Check info.xml and [project//data] folder <%s> <<<\n", + exec_path); + printf("Aborting comparison!\n"); + printf(" FAILED\n"); + error_count++; + + if (src_fp) { + fclose(src_fp); + } + + if (ref_fp) { + fclose(ref_fp); + } + } else { + if (FOPEN_FAIL(FOPEN(ref_fp, ref_file_path, "rb"))) { + printf("compareBin2Bin unable to open ref_file: %s\n", + ref_file_path); + error_count = 1; + } + + if (src_fp && ref_fp) { + src_buffer = reinterpret_cast(malloc(nelements * sizeof(float))); + ref_buffer = reinterpret_cast(malloc(nelements * sizeof(float))); + + printf( + "> compareBin2Bin nelements=%d, epsilon=%4.2f," + " threshold=%4.2f\n", + nelements, epsilon, threshold); + fsize = fread(src_buffer, sizeof(float), nelements, src_fp); + printf(" src_file <%s>, size=%d bytes\n", src_file, + static_cast(fsize * sizeof(float))); + fsize = fread(ref_buffer, sizeof(float), nelements, ref_fp); + printf(" ref_file <%s>, size=%d bytes\n", ref_file_path, + static_cast(fsize * sizeof(float))); + + if (!compareDataAsFloatThreshold( + ref_buffer, src_buffer, nelements, epsilon, threshold)) { + error_count++; + } + + fclose(src_fp); + fclose(ref_fp); + + free(src_buffer); + free(ref_buffer); + } else { + if (src_fp) { + fclose(src_fp); + } + + if (ref_fp) { + fclose(ref_fp); + } + } + } + + if (error_count == 0) { + printf(" OK\n"); + } else { + printf(" FAILURE: %d errors...\n", (unsigned int)error_count); + } + + return (error_count == 0); // returns true if all pixels pass +} + +inline bool sdkCompareL2fe(const float *reference, const float *data, + const unsigned int len, const float epsilon) { + assert(epsilon >= 0); + + float error = 0; + float ref = 0; + + for (unsigned int i = 0; i < len; ++i) { + float diff = reference[i] - data[i]; + error += diff * diff; + ref += reference[i] * reference[i]; + } + + float normRef = sqrtf(ref); + + if (fabs(ref) < 1e-7) { +#ifdef _DEBUG + std::cerr << "ERROR, reference l2-norm is 0\n"; +#endif + return false; + } + + float normError = sqrtf(error); + error = normError / normRef; + bool result = error < epsilon; +#ifdef _DEBUG + + if (!result) { + std::cerr << "ERROR, l2-norm error " << error << " is greater than epsilon " + << epsilon << "\n"; + } + +#endif + + return result; +} + +inline bool sdkLoadPPMub(const char *file, unsigned char **data, + unsigned int *w, unsigned int *h) { + unsigned int channels; + return __loadPPM(file, data, w, h, &channels); +} + +inline bool sdkLoadPPM4ub(const char *file, unsigned char **data, + unsigned int *w, unsigned int *h) { + unsigned char *idata = 0; + unsigned int channels; + + if (__loadPPM(file, &idata, w, h, &channels)) { + // pad 4th component + int size = *w * *h; + // keep the original pointer + unsigned char *idata_orig = idata; + *data = (unsigned char *)malloc(sizeof(unsigned char) * size * 4); + unsigned char *ptr = *data; + + for (int i = 0; i < size; i++) { + *ptr++ = *idata++; + *ptr++ = *idata++; + *ptr++ = *idata++; + *ptr++ = 0; + } + + free(idata_orig); + return true; + } else { + free(idata); + return false; + } +} + +inline bool sdkComparePPM(const char *src_file, const char *ref_file, + const float epsilon, const float threshold, + bool verboseErrors) { + unsigned char *src_data, *ref_data; + uint64_t error_count = 0; + unsigned int ref_width, ref_height; + unsigned int src_width, src_height; + + if (src_file == NULL || ref_file == NULL) { + if (verboseErrors) { + std::cerr << "PPMvsPPM: src_file or ref_file is NULL." + " Aborting comparison\n"; + } + + return false; + } + + if (verboseErrors) { + std::cerr << "> Compare (a)rendered: <" << src_file << ">\n"; + std::cerr << "> (b)reference: <" << ref_file << ">\n"; + } + + if (sdkLoadPPM4ub(ref_file, &ref_data, &ref_width, &ref_height) != true) { + if (verboseErrors) { + std::cerr << "PPMvsPPM: unable to load ref image file: " << ref_file + << "\n"; + } + + return false; + } + + if (sdkLoadPPM4ub(src_file, &src_data, &src_width, &src_height) != true) { + std::cerr << "PPMvsPPM: unable to load src image file: " << src_file + << "\n"; + return false; + } + + if (src_height != ref_height || src_width != ref_width) { + if (verboseErrors) { + std::cerr << "PPMvsPPM: source and ref size mismatch (" << src_width + << "," << src_height << ")vs(" << ref_width << "," << ref_height + << ")\n"; + } + } + + if (verboseErrors) { + std::cerr << "PPMvsPPM: comparing images size (" << src_width << "," + << src_height << ") epsilon(" << epsilon << "), threshold(" + << threshold * 100 << "%)\n"; + } + + if (compareData(ref_data, src_data, src_width * src_height * 4, epsilon, + threshold) == false) { + error_count = 1; + } + + if (error_count == 0) { + if (verboseErrors) { + std::cerr << " OK\n\n"; + } + } else { + if (verboseErrors) { + std::cerr << " FAILURE! " << error_count << " errors...\n\n"; + } + } + + // returns true if all pixels pass + return (error_count == 0) ? true : false; +} + +inline bool sdkComparePGM(const char *src_file, const char *ref_file, + const float epsilon, const float threshold, + bool verboseErrors) { + unsigned char *src_data = 0, *ref_data = 0; + uint64_t error_count = 0; + unsigned int ref_width, ref_height; + unsigned int src_width, src_height; + + if (src_file == NULL || ref_file == NULL) { + if (verboseErrors) { + std::cerr << "PGMvsPGM: src_file or ref_file is NULL." + " Aborting comparison\n"; + } + + return false; + } + + if (verboseErrors) { + std::cerr << "> Compare (a)rendered: <" << src_file << ">\n"; + std::cerr << "> (b)reference: <" << ref_file << ">\n"; + } + + if (sdkLoadPPMub(ref_file, &ref_data, &ref_width, &ref_height) != true) { + if (verboseErrors) { + std::cerr << "PGMvsPGM: unable to load ref image file: " << ref_file + << "\n"; + } + + return false; + } + + if (sdkLoadPPMub(src_file, &src_data, &src_width, &src_height) != true) { + std::cerr << "PGMvsPGM: unable to load src image file: " << src_file + << "\n"; + return false; + } + + if (src_height != ref_height || src_width != ref_width) { + if (verboseErrors) { + std::cerr << "PGMvsPGM: source and ref size mismatch (" << src_width + << "," << src_height << ")vs(" << ref_width << "," << ref_height + << ")\n"; + } + } + + if (verboseErrors) + std::cerr << "PGMvsPGM: comparing images size (" << src_width << "," + << src_height << ") epsilon(" << epsilon << "), threshold(" + << threshold * 100 << "%)\n"; + + if (compareData(ref_data, src_data, src_width * src_height, epsilon, + threshold) == false) { + error_count = 1; + } + + if (error_count == 0) { + if (verboseErrors) { + std::cerr << " OK\n\n"; + } + } else { + if (verboseErrors) { + std::cerr << " FAILURE! " << error_count << " errors...\n\n"; + } + } + + // returns true if all pixels pass + return (error_count == 0) ? true : false; +} + +#endif // COMMON_HELPER_IMAGE_H_ diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_string.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_string.h new file mode 100644 index 0000000..7920060 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_string.h @@ -0,0 +1,428 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// These are helper functions for the SDK samples (string parsing, timers, etc) +#ifndef COMMON_HELPER_STRING_H_ +#define COMMON_HELPER_STRING_H_ + +#include +#include +#include +#include + +#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) +#ifndef _CRT_SECURE_NO_DEPRECATE +#define _CRT_SECURE_NO_DEPRECATE +#endif +#ifndef STRCASECMP +#define STRCASECMP _stricmp +#endif +#ifndef STRNCASECMP +#define STRNCASECMP _strnicmp +#endif +#ifndef STRCPY +#define STRCPY(sFilePath, nLength, sPath) strcpy_s(sFilePath, nLength, sPath) +#endif + +#ifndef FOPEN +#define FOPEN(fHandle, filename, mode) fopen_s(&fHandle, filename, mode) +#endif +#ifndef FOPEN_FAIL +#define FOPEN_FAIL(result) (result != 0) +#endif +#ifndef SSCANF +#define SSCANF sscanf_s +#endif +#ifndef SPRINTF +#define SPRINTF sprintf_s +#endif +#else // Linux Includes +#include +#include + +#ifndef STRCASECMP +#define STRCASECMP strcasecmp +#endif +#ifndef STRNCASECMP +#define STRNCASECMP strncasecmp +#endif +#ifndef STRCPY +#define STRCPY(sFilePath, nLength, sPath) strcpy(sFilePath, sPath) +#endif + +#ifndef FOPEN +#define FOPEN(fHandle, filename, mode) (fHandle = fopen(filename, mode)) +#endif +#ifndef FOPEN_FAIL +#define FOPEN_FAIL(result) (result == NULL) +#endif +#ifndef SSCANF +#define SSCANF sscanf +#endif +#ifndef SPRINTF +#define SPRINTF sprintf +#endif +#endif + +#ifndef EXIT_WAIVED +#define EXIT_WAIVED 2 +#endif + +// CUDA Utility Helper Functions +inline int stringRemoveDelimiter(char delimiter, const char *string) { + int string_start = 0; + + while (string[string_start] == delimiter) { + string_start++; + } + + if (string_start >= static_cast(strlen(string) - 1)) { + return 0; + } + + return string_start; +} + +inline int getFileExtension(char *filename, char **extension) { + int string_length = static_cast(strlen(filename)); + + while (filename[string_length--] != '.') { + if (string_length == 0) break; + } + + if (string_length > 0) string_length += 2; + + if (string_length == 0) + *extension = NULL; + else + *extension = &filename[string_length]; + + return string_length; +} + +inline bool checkCmdLineFlag(const int argc, const char **argv, + const char *string_ref) { + bool bFound = false; + + if (argc >= 1) { + for (int i = 1; i < argc; i++) { + int string_start = stringRemoveDelimiter('-', argv[i]); + const char *string_argv = &argv[i][string_start]; + + const char *equal_pos = strchr(string_argv, '='); + int argv_length = static_cast( + equal_pos == 0 ? strlen(string_argv) : equal_pos - string_argv); + + int length = static_cast(strlen(string_ref)); + + if (length == argv_length && + !STRNCASECMP(string_argv, string_ref, length)) { + bFound = true; + continue; + } + } + } + + return bFound; +} + +// This function wraps the CUDA Driver API into a template function +template +inline bool getCmdLineArgumentValue(const int argc, const char **argv, + const char *string_ref, T *value) { + bool bFound = false; + + if (argc >= 1) { + for (int i = 1; i < argc; i++) { + int string_start = stringRemoveDelimiter('-', argv[i]); + const char *string_argv = &argv[i][string_start]; + int length = static_cast(strlen(string_ref)); + + if (!STRNCASECMP(string_argv, string_ref, length)) { + if (length + 1 <= static_cast(strlen(string_argv))) { + int auto_inc = (string_argv[length] == '=') ? 1 : 0; + *value = (T)atoi(&string_argv[length + auto_inc]); + } + + bFound = true; + i = argc; + } + } + } + + return bFound; +} + +inline int getCmdLineArgumentInt(const int argc, const char **argv, + const char *string_ref) { + bool bFound = false; + int value = -1; + + if (argc >= 1) { + for (int i = 1; i < argc; i++) { + int string_start = stringRemoveDelimiter('-', argv[i]); + const char *string_argv = &argv[i][string_start]; + int length = static_cast(strlen(string_ref)); + + if (!STRNCASECMP(string_argv, string_ref, length)) { + if (length + 1 <= static_cast(strlen(string_argv))) { + int auto_inc = (string_argv[length] == '=') ? 1 : 0; + value = atoi(&string_argv[length + auto_inc]); + } else { + value = 0; + } + + bFound = true; + continue; + } + } + } + + if (bFound) { + return value; + } else { + return 0; + } +} + +inline float getCmdLineArgumentFloat(const int argc, const char **argv, + const char *string_ref) { + bool bFound = false; + float value = -1; + + if (argc >= 1) { + for (int i = 1; i < argc; i++) { + int string_start = stringRemoveDelimiter('-', argv[i]); + const char *string_argv = &argv[i][string_start]; + int length = static_cast(strlen(string_ref)); + + if (!STRNCASECMP(string_argv, string_ref, length)) { + if (length + 1 <= static_cast(strlen(string_argv))) { + int auto_inc = (string_argv[length] == '=') ? 1 : 0; + value = static_cast(atof(&string_argv[length + auto_inc])); + } else { + value = 0.f; + } + + bFound = true; + continue; + } + } + } + + if (bFound) { + return value; + } else { + return 0; + } +} + +inline bool getCmdLineArgumentString(const int argc, const char **argv, + const char *string_ref, + char **string_retval) { + bool bFound = false; + + if (argc >= 1) { + for (int i = 1; i < argc; i++) { + int string_start = stringRemoveDelimiter('-', argv[i]); + char *string_argv = const_cast(&argv[i][string_start]); + int length = static_cast(strlen(string_ref)); + + if (!STRNCASECMP(string_argv, string_ref, length)) { + *string_retval = &string_argv[length]; + bFound = true; + continue; + } + } + } + + if (!bFound) { + *string_retval = NULL; + } + + return bFound; +} + +////////////////////////////////////////////////////////////////////////////// +//! Find the path for a file assuming that +//! files are found in the searchPath. +//! +//! @return the path if succeeded, otherwise 0 +//! @param filename name of the file +//! @param executable_path optional absolute path of the executable +////////////////////////////////////////////////////////////////////////////// +inline char *sdkFindFilePath(const char *filename, + const char *executable_path) { + // defines a variable that is replaced with the name of the + // executable + + // Typical relative search paths to locate needed companion files (e.g. sample + // input data, or JIT source files) The origin for the relative search may be + // the .exe file, a .bat file launching an .exe, a browser .exe launching the + // .exe or .bat, etc + const char *searchPath[] = { + "./", // same dir + "./data/", // same dir + + "../../../../Samples//", // up 4 in tree + "../../../Samples//", // up 3 in tree + "../../Samples//", // up 2 in tree + + "../../../../Samples//data/", // up 4 in tree + "../../../Samples//data/", // up 3 in tree + "../../Samples//data/", // up 2 in tree + + "../../../../Samples/0_Introduction//", // up 4 in tree + "../../../Samples/0_Introduction//", // up 3 in tree + "../../Samples/0_Introduction//", // up 2 in tree + + "../../../../Samples/1_Utilities//", // up 4 in tree + "../../../Samples/1_Utilities//", // up 3 in tree + "../../Samples/1_Utilities//", // up 2 in tree + + "../../../../Samples/2_Concepts_and_Techniques//", // up 4 in tree + "../../../Samples/2_Concepts_and_Techniques//", // up 3 in tree + "../../Samples/2_Concepts_and_Techniques//", // up 2 in tree + + "../../../../Samples/3_CUDA_Features//", // up 4 in tree + "../../../Samples/3_CUDA_Features//", // up 3 in tree + "../../Samples/3_CUDA_Features//", // up 2 in tree + + "../../../../Samples/4_CUDA_Libraries//", // up 4 in tree + "../../../Samples/4_CUDA_Libraries//", // up 3 in tree + "../../Samples/4_CUDA_Libraries//", // up 2 in tree + + "../../../../Samples/5_Domain_Specific//", // up 4 in tree + "../../../Samples/5_Domain_Specific//", // up 3 in tree + "../../Samples/5_Domain_Specific//", // up 2 in tree + + "../../../../Samples/6_Performance//", // up 4 in tree + "../../../Samples/6_Performance//", // up 3 in tree + "../../Samples/6_Performance//", // up 2 in tree + + "../../../../Samples/0_Introduction//data/", // up 4 in tree + "../../../Samples/0_Introduction//data/", // up 3 in tree + "../../Samples/0_Introduction//data/", // up 2 in tree + + "../../../../Samples/1_Utilities//data/", // up 4 in tree + "../../../Samples/1_Utilities//data/", // up 3 in tree + "../../Samples/1_Utilities//data/", // up 2 in tree + + "../../../../Samples/2_Concepts_and_Techniques//data/", // up 4 in tree + "../../../Samples/2_Concepts_and_Techniques//data/", // up 3 in tree + "../../Samples/2_Concepts_and_Techniques//data/", // up 2 in tree + + "../../../../Samples/3_CUDA_Features//data/", // up 4 in tree + "../../../Samples/3_CUDA_Features//data/", // up 3 in tree + "../../Samples/3_CUDA_Features//data/", // up 2 in tree + + "../../../../Samples/4_CUDA_Libraries//data/", // up 4 in tree + "../../../Samples/4_CUDA_Libraries//data/", // up 3 in tree + "../../Samples/4_CUDA_Libraries//data/", // up 2 in tree + + "../../../../Samples/5_Domain_Specific//data/", // up 4 in tree + "../../../Samples/5_Domain_Specific//data/", // up 3 in tree + "../../Samples/5_Domain_Specific//data/", // up 2 in tree + + "../../../../Samples/6_Performance//data/", // up 4 in tree + "../../../Samples/6_Performance//data/", // up 3 in tree + "../../Samples/6_Performance//data/", // up 2 in tree + + "../../../../Common/data/", // up 4 in tree + "../../../Common/data/", // up 3 in tree + "../../Common/data/" // up 2 in tree + }; + + // Extract the executable name + std::string executable_name; + + if (executable_path != 0) { + executable_name = std::string(executable_path); + +#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) + // Windows path delimiter + size_t delimiter_pos = executable_name.find_last_of('\\'); + executable_name.erase(0, delimiter_pos + 1); + + if (executable_name.rfind(".exe") != std::string::npos) { + // we strip .exe, only if the .exe is found + executable_name.resize(executable_name.size() - 4); + } + +#else + // Linux & OSX path delimiter + size_t delimiter_pos = executable_name.find_last_of('/'); + executable_name.erase(0, delimiter_pos + 1); +#endif + } + + // Loop over all search paths and return the first hit + for (unsigned int i = 0; i < sizeof(searchPath) / sizeof(char *); ++i) { + std::string path(searchPath[i]); + size_t executable_name_pos = path.find(""); + + // If there is executable_name variable in the searchPath + // replace it with the value + if (executable_name_pos != std::string::npos) { + if (executable_path != 0) { + path.replace(executable_name_pos, strlen(""), + executable_name); + } else { + // Skip this path entry if no executable argument is given + continue; + } + } + +#ifdef _DEBUG + printf("sdkFindFilePath <%s> in %s\n", filename, path.c_str()); +#endif + + // Test if the file exists + path.append(filename); + FILE *fp; + FOPEN(fp, path.c_str(), "rb"); + + if (fp != NULL) { + fclose(fp); + // File found + // returning an allocated array here for backwards compatibility reasons + char *file_path = reinterpret_cast(malloc(path.length() + 1)); + STRCPY(file_path, path.length() + 1, path.c_str()); + return file_path; + } + + if (fp) { + fclose(fp); + } + } + + // File not found + printf("\nerror: sdkFindFilePath: file <%s> not found!\n", filename); + return 0; +} + +#endif // COMMON_HELPER_STRING_H_ diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_timer.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_timer.h new file mode 100644 index 0000000..2fe3207 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_timer.h @@ -0,0 +1,465 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// Helper Timing Functions +#ifndef COMMON_HELPER_TIMER_H_ +#define COMMON_HELPER_TIMER_H_ + +#ifndef EXIT_WAIVED +#define EXIT_WAIVED 2 +#endif + +// includes, system +#include + +// includes, project +#include + +// Definition of the StopWatch Interface, this is used if we don't want to use +// the CUT functions But rather in a self contained class interface +class StopWatchInterface { + public: + StopWatchInterface() {} + virtual ~StopWatchInterface() {} + + public: + //! Start time measurement + virtual void start() = 0; + + //! Stop time measurement + virtual void stop() = 0; + + //! Reset time counters to zero + virtual void reset() = 0; + + //! Time in msec. after start. If the stop watch is still running (i.e. there + //! was no call to stop()) then the elapsed time is returned, otherwise the + //! time between the last start() and stop call is returned + virtual float getTime() = 0; + + //! Mean time to date based on the number of times the stopwatch has been + //! _stopped_ (ie finished sessions) and the current total time + virtual float getAverageTime() = 0; +}; + +////////////////////////////////////////////////////////////////// +// Begin Stopwatch timer class definitions for all OS platforms // +////////////////////////////////////////////////////////////////// +#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) +// includes, system +#define WINDOWS_LEAN_AND_MEAN +#include +#undef min +#undef max + +//! Windows specific implementation of StopWatch +class StopWatchWin : public StopWatchInterface { + public: + //! Constructor, default + StopWatchWin() + : start_time(), + end_time(), + diff_time(0.0f), + total_time(0.0f), + running(false), + clock_sessions(0), + freq(0), + freq_set(false) { + if (!freq_set) { + // helper variable + LARGE_INTEGER temp; + + // get the tick frequency from the OS + QueryPerformanceFrequency(reinterpret_cast(&temp)); + + // convert to type in which it is needed + freq = (static_cast(temp.QuadPart)) / 1000.0; + + // rememeber query + freq_set = true; + } + } + + // Destructor + ~StopWatchWin() {} + + public: + //! Start time measurement + inline void start(); + + //! Stop time measurement + inline void stop(); + + //! Reset time counters to zero + inline void reset(); + + //! Time in msec. after start. If the stop watch is still running (i.e. there + //! was no call to stop()) then the elapsed time is returned, otherwise the + //! time between the last start() and stop call is returned + inline float getTime(); + + //! Mean time to date based on the number of times the stopwatch has been + //! _stopped_ (ie finished sessions) and the current total time + inline float getAverageTime(); + + private: + // member variables + + //! Start of measurement + LARGE_INTEGER start_time; + //! End of measurement + LARGE_INTEGER end_time; + + //! Time difference between the last start and stop + float diff_time; + + //! TOTAL time difference between starts and stops + float total_time; + + //! flag if the stop watch is running + bool running; + + //! Number of times clock has been started + //! and stopped to allow averaging + int clock_sessions; + + //! tick frequency + double freq; + + //! flag if the frequency has been set + bool freq_set; +}; + +// functions, inlined + +//////////////////////////////////////////////////////////////////////////////// +//! Start time measurement +//////////////////////////////////////////////////////////////////////////////// +inline void StopWatchWin::start() { + QueryPerformanceCounter(reinterpret_cast(&start_time)); + running = true; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Stop time measurement and increment add to the current diff_time summation +//! variable. Also increment the number of times this clock has been run. +//////////////////////////////////////////////////////////////////////////////// +inline void StopWatchWin::stop() { + QueryPerformanceCounter(reinterpret_cast(&end_time)); + diff_time = static_cast(((static_cast(end_time.QuadPart) - + static_cast(start_time.QuadPart)) / + freq)); + + total_time += diff_time; + clock_sessions++; + running = false; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Reset the timer to 0. Does not change the timer running state but does +//! recapture this point in time as the current start time if it is running. +//////////////////////////////////////////////////////////////////////////////// +inline void StopWatchWin::reset() { + diff_time = 0; + total_time = 0; + clock_sessions = 0; + + if (running) { + QueryPerformanceCounter(reinterpret_cast(&start_time)); + } +} + +//////////////////////////////////////////////////////////////////////////////// +//! Time in msec. after start. If the stop watch is still running (i.e. there +//! was no call to stop()) then the elapsed time is returned added to the +//! current diff_time sum, otherwise the current summed time difference alone +//! is returned. +//////////////////////////////////////////////////////////////////////////////// +inline float StopWatchWin::getTime() { + // Return the TOTAL time to date + float retval = total_time; + + if (running) { + LARGE_INTEGER temp; + QueryPerformanceCounter(reinterpret_cast(&temp)); + retval += static_cast(((static_cast(temp.QuadPart) - + static_cast(start_time.QuadPart)) / + freq)); + } + + return retval; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Time in msec. for a single run based on the total number of COMPLETED runs +//! and the total time. +//////////////////////////////////////////////////////////////////////////////// +inline float StopWatchWin::getAverageTime() { + return (clock_sessions > 0) ? (total_time / clock_sessions) : 0.0f; +} +#else +// Declarations for Stopwatch on Linux and Mac OSX +// includes, system +#include +#include + +//! Windows specific implementation of StopWatch +class StopWatchLinux : public StopWatchInterface { + public: + //! Constructor, default + StopWatchLinux() + : start_time(), + diff_time(0.0), + total_time(0.0), + running(false), + clock_sessions(0) {} + + // Destructor + virtual ~StopWatchLinux() {} + + public: + //! Start time measurement + inline void start(); + + //! Stop time measurement + inline void stop(); + + //! Reset time counters to zero + inline void reset(); + + //! Time in msec. after start. If the stop watch is still running (i.e. there + //! was no call to stop()) then the elapsed time is returned, otherwise the + //! time between the last start() and stop call is returned + inline float getTime(); + + //! Mean time to date based on the number of times the stopwatch has been + //! _stopped_ (ie finished sessions) and the current total time + inline float getAverageTime(); + + private: + // helper functions + + //! Get difference between start time and current time + inline float getDiffTime(); + + private: + // member variables + + //! Start of measurement + struct timeval start_time; + + //! Time difference between the last start and stop + float diff_time; + + //! TOTAL time difference between starts and stops + float total_time; + + //! flag if the stop watch is running + bool running; + + //! Number of times clock has been started + //! and stopped to allow averaging + int clock_sessions; +}; + +// functions, inlined + +//////////////////////////////////////////////////////////////////////////////// +//! Start time measurement +//////////////////////////////////////////////////////////////////////////////// +inline void StopWatchLinux::start() { + gettimeofday(&start_time, 0); + running = true; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Stop time measurement and increment add to the current diff_time summation +//! variable. Also increment the number of times this clock has been run. +//////////////////////////////////////////////////////////////////////////////// +inline void StopWatchLinux::stop() { + diff_time = getDiffTime(); + total_time += diff_time; + running = false; + clock_sessions++; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Reset the timer to 0. Does not change the timer running state but does +//! recapture this point in time as the current start time if it is running. +//////////////////////////////////////////////////////////////////////////////// +inline void StopWatchLinux::reset() { + diff_time = 0; + total_time = 0; + clock_sessions = 0; + + if (running) { + gettimeofday(&start_time, 0); + } +} + +//////////////////////////////////////////////////////////////////////////////// +//! Time in msec. after start. If the stop watch is still running (i.e. there +//! was no call to stop()) then the elapsed time is returned added to the +//! current diff_time sum, otherwise the current summed time difference alone +//! is returned. +//////////////////////////////////////////////////////////////////////////////// +inline float StopWatchLinux::getTime() { + // Return the TOTAL time to date + float retval = total_time; + + if (running) { + retval += getDiffTime(); + } + + return retval; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Time in msec. for a single run based on the total number of COMPLETED runs +//! and the total time. +//////////////////////////////////////////////////////////////////////////////// +inline float StopWatchLinux::getAverageTime() { + return (clock_sessions > 0) ? (total_time / clock_sessions) : 0.0f; +} +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +inline float StopWatchLinux::getDiffTime() { + struct timeval t_time; + gettimeofday(&t_time, 0); + + // time difference in milli-seconds + return static_cast(1000.0 * (t_time.tv_sec - start_time.tv_sec) + + (0.001 * (t_time.tv_usec - start_time.tv_usec))); +} +#endif // WIN32 + +//////////////////////////////////////////////////////////////////////////////// +//! Timer functionality exported + +//////////////////////////////////////////////////////////////////////////////// +//! Create a new timer +//! @return true if a time has been created, otherwise false +//! @param name of the new timer, 0 if the creation failed +//////////////////////////////////////////////////////////////////////////////// +inline bool sdkCreateTimer(StopWatchInterface **timer_interface) { +// printf("sdkCreateTimer called object %08x\n", (void *)*timer_interface); +#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) + *timer_interface = reinterpret_cast(new StopWatchWin()); +#else + *timer_interface = + reinterpret_cast(new StopWatchLinux()); +#endif + return (*timer_interface != NULL) ? true : false; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Delete a timer +//! @return true if a time has been deleted, otherwise false +//! @param name of the timer to delete +//////////////////////////////////////////////////////////////////////////////// +inline bool sdkDeleteTimer(StopWatchInterface **timer_interface) { + // printf("sdkDeleteTimer called object %08x\n", (void *)*timer_interface); + if (*timer_interface) { + delete *timer_interface; + *timer_interface = NULL; + } + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Start the time with name \a name +//! @param name name of the timer to start +//////////////////////////////////////////////////////////////////////////////// +inline bool sdkStartTimer(StopWatchInterface **timer_interface) { + // printf("sdkStartTimer called object %08x\n", (void *)*timer_interface); + if (*timer_interface) { + (*timer_interface)->start(); + } + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Stop the time with name \a name. Does not reset. +//! @param name name of the timer to stop +//////////////////////////////////////////////////////////////////////////////// +inline bool sdkStopTimer(StopWatchInterface **timer_interface) { + // printf("sdkStopTimer called object %08x\n", (void *)*timer_interface); + if (*timer_interface) { + (*timer_interface)->stop(); + } + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Resets the timer's counter. +//! @param name name of the timer to reset. +//////////////////////////////////////////////////////////////////////////////// +inline bool sdkResetTimer(StopWatchInterface **timer_interface) { + // printf("sdkResetTimer called object %08x\n", (void *)*timer_interface); + if (*timer_interface) { + (*timer_interface)->reset(); + } + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Return the average time for timer execution as the total time +//! for the timer dividied by the number of completed (stopped) runs the timer +//! has made. +//! Excludes the current running time if the timer is currently running. +//! @param name name of the timer to return the time of +//////////////////////////////////////////////////////////////////////////////// +inline float sdkGetAverageTimerValue(StopWatchInterface **timer_interface) { + // printf("sdkGetAverageTimerValue called object %08x\n", (void + // *)*timer_interface); + if (*timer_interface) { + return (*timer_interface)->getAverageTime(); + } else { + return 0.0f; + } +} + +//////////////////////////////////////////////////////////////////////////////// +//! Total execution time for the timer over all runs since the last reset +//! or timer creation. +//! @param name name of the timer to obtain the value of. +//////////////////////////////////////////////////////////////////////////////// +inline float sdkGetTimerValue(StopWatchInterface **timer_interface) { + // printf("sdkGetTimerValue called object %08x\n", (void *)*timer_interface); + if (*timer_interface) { + return (*timer_interface)->getTime(); + } else { + return 0.0f; + } +} + +#endif // COMMON_HELPER_TIMER_H_ diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/nccl_util.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/nccl_util.h new file mode 100644 index 0000000..9134c2f --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/nccl_util.h @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define USE_CONFIG +#define SERVER_IP "192.168.0.208" +#define SERVER_PORT 15916 +#define BUFFER_SIZE 8 +#define GPUS_NUM 2 +#define CLIENTS_NUM 1 + +#define CUDACHECK(cmd) \ + do { \ + cudaError_t e = cmd; \ + if (e != cudaSuccess) { \ + printf("Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \ + cudaGetErrorString(e)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define NCCLCHECK(cmd) \ + do { \ + ncclResult_t r = cmd; \ + if (r != ncclSuccess) { \ + printf("Failed, NCCL error %s:%d '%s'\n", __FILE__, __LINE__, \ + ncclGetErrorString(r)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +static void PrintBuffer(void** devBuff) { + float* hostBuff = (float*)malloc(BUFFER_SIZE * sizeof(float)); + for (int devID = 0; devID < GPUS_NUM; devID++) { + printf("GPU[%d]", devID); + CUDACHECK(cudaSetDevice(devID)); + CUDACHECK(cudaMemcpy(hostBuff, devBuff[devID], BUFFER_SIZE * sizeof(float), cudaMemcpyDeviceToHost)); + + for(int i = 0; i < BUFFER_SIZE; i++) { + printf("%f ", hostBuff[i]); + } + printf("\n"); + } + free(hostBuff); +} + +static void PrintBuffer(void** devBuff, size_t cnt) { + float* hostBuff = (float*)malloc(cnt * sizeof(float)); + for (int devID = 0; devID < GPUS_NUM; devID++) { + printf("GPU[%d]", devID); + CUDACHECK(cudaSetDevice(devID)); + CUDACHECK(cudaMemcpy(hostBuff, devBuff[devID], cnt * sizeof(float), cudaMemcpyDeviceToHost)); + + for(int i = 0; i < cnt; i++) { + printf("%f ", hostBuff[i]); + } + printf("\n"); + } + free(hostBuff); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNN.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNN.cu new file mode 100644 index 0000000..08a9e9a --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNN.cu @@ -0,0 +1,339 @@ +/** +* Initialized CUDA and cuDNN context. +* Input data initialized: +* 0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 +* Input Data prepared: tensors allocated and initialized. +* Extra parameters and workspace for batch normalization allocated. +* Forward and backward propagation completed. +* Batch Normalization Output: +* -1.62621 -1.62619 -1.62617 -1.62615 -1.62613 -1.62611 -1.6261 -1.62608 -1.62606 -1.62604 +* Input Gradient: +* 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 +*/ +#include +#include +#include +#include + +#define checkCUDNN(expression) \ +{ \ + cudnnStatus_t status = (expression); \ + if (status != CUDNN_STATUS_SUCCESS) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudnnGetErrorString(status) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +#define checkCUDA(expression) \ +{ \ + cudaError_t error = (expression); \ + if (error != cudaSuccess) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudaGetErrorString(error) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +// CUDA内核，用于初始化输入数据为等差数列 +__global__ void initArithmeticSequence(float *data, int n, float start, float step) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx < n) { + data[idx] = start + step * idx; + } +} + +int main() { + cudnnHandle_t cudnn; + checkCUDNN(cudnnCreate(&cudnn)); // 初始化cuDNN + + cudaStream_t stream; + checkCUDA(cudaStreamCreate(&stream)); // 初始化CUDA流 + checkCUDNN(cudnnSetStream(cudnn, stream)); // 设置cuDNN流 + + // 数据维度和批量标准化参数 + const int batch_size = 64; // 批大小 + const int channels = 3; // 输入通道数 + const int height = 32; // 图像高度 + const int width = 32; // 图像宽度 + const float epsilon = 1e-5f; // 用于批量标准化的epsilon值 + const float momentum = 0.9f; // 用于移动平均的动量值 + + std::cout << "Initialized CUDA and cuDNN context." << std::endl; + + // 设定输入输出张量的维度 + const int tensorDims = 4; + int tensorDimsA[tensorDims] = {batch_size, channels, height, width}; // 定义维度大小 + int strideA[tensorDims] = {channels * height * width, height * width, width, 1}; // 定义步长 + + // 创建张量描述符 + cudnnTensorDescriptor_t inputDescriptor, outputDescriptor, dyDescriptor, dxDescriptor; + checkCUDNN(cudnnCreateTensorDescriptor(&inputDescriptor)); + checkCUDNN(cudnnCreateTensorDescriptor(&outputDescriptor)); + checkCUDNN(cudnnCreateTensorDescriptor(&dyDescriptor)); // 输出梯度描述符 + checkCUDNN(cudnnCreateTensorDescriptor(&dxDescriptor)); // 输入梯度描述符 + + // 设置张量描述符的格式和维度 + checkCUDNN(cudnnSetTensorNdDescriptor(inputDescriptor, + CUDNN_DATA_FLOAT, + tensorDims, + tensorDimsA, + strideA)); + // checkCUDNN(cudnnSetTensor4dDescriptor(outputDescriptor, + // CUDNN_TENSOR_NCHW, + // CUDNN_DATA_FLOAT, + // batch_size, + // channels, + // height, + // width)); + checkCUDNN(cudnnSetTensorNdDescriptorEx(outputDescriptor, + CUDNN_TENSOR_NCHW, + CUDNN_DATA_FLOAT, + tensorDims, + tensorDimsA)); + checkCUDNN(cudnnSetTensor4dDescriptor(dyDescriptor, + CUDNN_TENSOR_NCHW, + CUDNN_DATA_FLOAT, + batch_size, + channels, + height, + width)); + checkCUDNN(cudnnSetTensor4dDescriptor(dxDescriptor, + CUDNN_TENSOR_NCHW, + CUDNN_DATA_FLOAT, + batch_size, + channels, + height, + width)); + + // 为输入数据、输出数据、批量标准化参数分配GPU内存 + float *d_input, *d_output, *d_output_grad, *d_input_grad; + float *d_scale, *d_bias; + + checkCUDA(cudaMalloc(&d_input, sizeof(float) * batch_size * channels * height * width)); + checkCUDA(cudaMalloc(&d_output, sizeof(float) * batch_size * channels * height * width)); + checkCUDA(cudaMalloc(&d_output_grad, sizeof(float) * batch_size * channels * height * width)); // 输出梯度 + checkCUDA(cudaMalloc(&d_input_grad, sizeof(float) * batch_size * channels * height * width)); // 输入梯度 + checkCUDA(cudaMalloc(&d_scale, sizeof(float) * channels)); + checkCUDA(cudaMalloc(&d_bias, sizeof(float) * channels)); + + // 初始化输入数据为等差数列 + int n = batch_size * channels * height * width; + float start = 0.0f; // 等差数列起始值 + float step = 0.1f; // 等差数列步长 + int threadsPerBlock = 256; + int blocksPerGrid = (n + threadsPerBlock - 1) / threadsPerBlock; + initArithmeticSequence<<>>(d_input, n, start, step); + cudaDeviceSynchronize(); // 确保CUDA内核执行完成 + float *h_input = new float[n]; + checkCUDA(cudaMemcpy(h_input, d_input, sizeof(float) * n, cudaMemcpyDeviceToHost)); + std::cout << "Input data initialized: " << std::endl; + for (int i = 0; i < 10; i++) { + std::cout << h_input[i] << " "; + } + std::cout << std::endl; + + // 初始化批量标准化参数 + std::vector h_scale(channels, 1.0f); // scale初始化为1 + std::vector h_bias(channels, 0.0f); // bias初始化为0 + + // 将初始化的参数从主机拷贝到设备 + checkCUDA(cudaMemcpy(d_scale, h_scale.data(), sizeof(float) * channels, cudaMemcpyHostToDevice)); + checkCUDA(cudaMemcpy(d_bias, h_bias.data(), sizeof(float) * channels, cudaMemcpyHostToDevice)); + + std::cout << "Input Data prepared: tensors allocated and initialized." << std::endl; + + // 前向传播和反向传播的额外参数 + float *d_runningMean, *d_runningVariance; + float *d_saveMean, *d_saveInvVariance; // 前向训练时保存的均值和逆方差 + float *d_bnScaleDiff, *d_bnBiasDiff; // 反向传播时计算的梯度 + void *d_workspace = nullptr; // 反向传播所需的工作空间 + size_t workspaceSize = 0; // 工作空间的大小 + size_t reserveSpaceSize = 0; // 保留空间的大小 + + // 分配额外参数的内存 + checkCUDA(cudaMalloc(&d_runningMean, sizeof(float) * channels)); + checkCUDA(cudaMalloc(&d_runningVariance, sizeof(float) * channels)); + checkCUDA(cudaMalloc(&d_saveMean, sizeof(float) * channels)); + checkCUDA(cudaMalloc(&d_saveInvVariance, sizeof(float) * channels)); + checkCUDA(cudaMalloc(&d_bnScaleDiff, sizeof(float) * channels)); + checkCUDA(cudaMalloc(&d_bnBiasDiff, sizeof(float) * channels)); + + // 初始化为0 + checkCUDA(cudaMemset(d_runningMean, 0, sizeof(float) * channels)); + checkCUDA(cudaMemset(d_runningVariance, 0, sizeof(float) * channels)); + + // 获取前向传播和反向传播所需的工作空间大小 + cudnnTensorDescriptor_t bnScaleBiasMeanVarDesc; + checkCUDNN(cudnnCreateTensorDescriptor(&bnScaleBiasMeanVarDesc)); + checkCUDNN(cudnnSetTensor4dDescriptor(bnScaleBiasMeanVarDesc, + CUDNN_TENSOR_NCHW, + CUDNN_DATA_FLOAT, + 1, channels, 1, 1)); + + cudnnBatchNormMode_t mode = CUDNN_BATCHNORM_SPATIAL_PERSISTENT; + cudnnBatchNormOps_t bnOps = CUDNN_BATCHNORM_OPS_BN; + // 假设与批量标准化操作不结合使用任何额外的激活函数 + cudnnActivationDescriptor_t activationDesc = nullptr; + + // 为前向训练计算工作空间大小 + checkCUDNN(cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize( + cudnn, // cudnn句柄 + mode, // 批量标准化模式 + bnOps, // 批量标准化操作 + inputDescriptor, // xDesc: 输入张量描述符 + nullptr, // zDesc: 偏置张量描述符（在这里不使用，因此为nullptr） + outputDescriptor, // yDesc: 输出张量描述符 + bnScaleBiasMeanVarDesc, // bnScaleBiasMeanVarDesc: 缩放、偏置、均值和方差的描述符 + activationDesc, // activationDesc: 激活描述符（在这里不使用，因此为nullptr） + &workspaceSize // sizeInBytes: 工作空间大小的指针 + )); + + // 为训练计算保留空间大小 + checkCUDNN(cudnnGetBatchNormalizationTrainingExReserveSpaceSize( + cudnn, // cudnn句柄 + mode, // 批量标准化模式 + bnOps, // 批量标准化操作 + activationDesc, // activationDesc: 激活描述符（在这里不使用，因此为nullptr） + inputDescriptor, // xDesc: 输入张量描述符 + &reserveSpaceSize // sizeInBytes: 保留空间大小的指针 + )); + + // 分配工作空间和保留空间 + void *d_reserveSpace = nullptr; // 用于反向传播的保留空间 + if (workspaceSize > 0) { + checkCUDA(cudaMalloc(&d_workspace, workspaceSize)); + std::cout << "Workspace for batch normalization allocated: " << workspaceSize << " bytes." << std::endl; + } + if (reserveSpaceSize > 0) { + checkCUDA(cudaMalloc(&d_reserveSpace, reserveSpaceSize)); + std::cout << "Reserve space for batch normalization allocated: " << reserveSpaceSize << " bytes." << std::endl; + } + + std::cout << "Extra parameters and workspace for batch normalization allocated." << std::endl; + + // 设置学习率参数，这些在实际训练中可能会变化或根据优化算法调整 + const float alpha = 1.0f; + const float beta = 0.0f; + + // 前向训练 + checkCUDNN(cudnnBatchNormalizationForwardTrainingEx( + cudnn, mode, bnOps, + &alpha, &beta, inputDescriptor, d_input, nullptr, nullptr, + outputDescriptor, d_output, bnScaleBiasMeanVarDesc, + d_scale, d_bias, momentum, d_runningMean, d_runningVariance, + epsilon, d_saveMean, d_saveInvVariance, activationDesc, + d_workspace, workspaceSize, d_reserveSpace, reserveSpaceSize + )); + + // 前向推理（用于评估或推理） + // 使用训练过程中计算得到的运行均值和方差 + checkCUDNN(cudnnBatchNormalizationForwardInference( + cudnn, mode, + &alpha, &beta, inputDescriptor, d_input, + outputDescriptor, d_output, bnScaleBiasMeanVarDesc, + d_scale, d_bias, d_runningMean, d_runningVariance, epsilon + )); + + // 执行批量标准化的反向传播 + const float alphaDataDiff = 1.0f; // 输出梯度缩放系数 + const float betaDataDiff = 0.0f; + const float alphaParamDiff = 1.0f; // 参数梯度缩放系数 + const float betaParamDiff = 0.0f; + const float outputGradVal = 1.0f; + checkCUDA(cudaMemset(d_output_grad, outputGradVal, sizeof(float) * batch_size * channels * height * width)); // 输出梯度初始化为1 + + size_t workspaceSizeBackward = 0; // 反向传播所需的工作空间大小 + checkCUDNN(cudnnGetBatchNormalizationBackwardExWorkspaceSize( + cudnn, // cudnn句柄 + mode, // 批量标准化模式 + bnOps, // 批量标准化操作 + inputDescriptor, // xDesc: 输入张量描述符 + outputDescriptor, // yDesc: 输出张量描述符 + dyDescriptor, // dyDesc: 输出梯度张量描述符 + nullptr, // dzDesc: 输入梯度张量描述符（在这里不使用，因此为nullptr） + dxDescriptor, // dxDesc: 输入梯度张量描述符 + bnScaleBiasMeanVarDesc, // bnScaleBiasMeanVarDesc: 缩放、偏置、均值和方差的描述符 + activationDesc, // activationDesc: 激活描述符（如果使用激活函数的话） + &workspaceSizeBackward // sizeInBytes: 指向存储工作空间大小的变量的指针 + )); + if (workspaceSizeBackward > 0) { + if (workspaceSizeBackward > workspaceSize) { + // 如果反向传播需要的工作空间比前向传播更大，重新分配 + cudaFree(d_workspace); // 释放之前分配的工作空间 + checkCUDA(cudaMalloc(&d_workspace, workspaceSizeBackward)); + } + // 如果前向传播的工作空间已经足够大，则无需重新分配 + std::cout << "Workspace for batch normalization backward allocated: " << workspaceSizeBackward << " bytes." << std::endl; + } + + checkCUDNN(cudnnBatchNormalizationBackwardEx( + cudnn, mode, bnOps, + &alphaDataDiff, &betaDataDiff, &alphaParamDiff, &betaParamDiff, + inputDescriptor, d_input, outputDescriptor, d_output, + dyDescriptor, d_output_grad, nullptr, nullptr, dxDescriptor, d_input_grad, + bnScaleBiasMeanVarDesc, d_scale, d_bias, d_bnScaleDiff, d_bnBiasDiff, + epsilon, d_saveMean, d_saveInvVariance, activationDesc, + d_workspace, workspaceSize, d_reserveSpace, reserveSpaceSize + )); + + std::cout << "Forward and backward propagation completed." << std::endl; + + // 将输出数据从GPU拷贝回主机 + float *h_output; + h_output = new float[batch_size * channels * height * width]; + checkCUDA(cudaMemcpy(h_output, d_output, sizeof(float) * batch_size * channels * height * width, cudaMemcpyDeviceToHost)); + + // 将输入梯度从GPU拷贝回主机 + float *h_input_grad; + h_input_grad = new float[batch_size * channels * height * width]; + checkCUDA(cudaMemcpy(h_input_grad, d_input_grad, sizeof(float) * batch_size * channels * height * width, cudaMemcpyDeviceToHost)); + + // 输出批量标准化的结果 + std::cout << "Batch Normalization Output:" << std::endl; + for (int i = 0; i < 10; i++) { + std::cout << h_output[i] << " "; + } + std::cout << std::endl; + // -1.62621 -1.62619 -1.62617 -1.62615 -1.62613 -1.62611 -1.6261 -1.62608 -1.62606 -1.6260 + + // 输出输入梯度 + std::cout << "Input Gradient:" << std::endl; + for (int i = 0; i < 10; i++) { + std::cout << h_input_grad[i] << " "; + } + std::cout << std::endl; + // 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 + + // 释放主机内存 + delete[] h_output; + delete[] h_input_grad; + + // 同步GPU并释放内存 + cudaDeviceSynchronize(); + cudaFree(d_input); + cudaFree(d_output); + cudaFree(d_output_grad); + cudaFree(d_input_grad); + cudaFree(d_scale); + cudaFree(d_bias); + cudaFree(d_runningMean); + cudaFree(d_runningVariance); + cudaFree(d_saveMean); + cudaFree(d_saveInvVariance); + cudaFree(d_bnScaleDiff); + cudaFree(d_bnBiasDiff); + cudaFree(d_workspace); + cudaFree(d_reserveSpace); + + // 清理cuDNN资源 + cudnnDestroyTensorDescriptor(inputDescriptor); + cudnnDestroyTensorDescriptor(outputDescriptor); + cudnnDestroyTensorDescriptor(dyDescriptor); + cudnnDestroyTensorDescriptor(dxDescriptor); + cudnnDestroyTensorDescriptor(bnScaleBiasMeanVarDesc); + cudnnDestroy(cudnn); + cudaStreamDestroy(stream); + + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNBackend.cu.bak b/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNBackend.cu.bak new file mode 100644 index 0000000..93e4e94 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNBackend.cu.bak @@ -0,0 +1,184 @@ +#include +#include +#include + +#define checkCUDNN(expression) \ +{ \ + cudnnStatus_t status = (expression); \ + if (status != CUDNN_STATUS_SUCCESS) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudnnGetErrorString(status) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +#define checkCUDA(expression) \ +{ \ + cudaError_t error = (expression); \ + if (error != cudaSuccess) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudaGetErrorString(error) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +int main() { + cudnnHandle_t handle; + checkCUDNN(cudnnCreate(&handle)); + + // Create a input tensor descriptors, convolution filter and output tensor descriptor + + cudnnBackendDescriptor_t xDesc; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_TENSOR_DESCRIPTOR, &xDesc)); + cudnnDataType_t dtype = CUDNN_DATA_FLOAT; + checkCUDNN(cudnnBackendSetAttribute(xDesc, CUDNN_ATTR_TENSOR_DATA_TYPE, CUDNN_TYPE_DATA_TYPE, 1, &dtype)); + // [n, g, c, d, h, w] = [32, 4, 16, 8, 28, 28] + int64_t n = 32, g = 4, c = 16, d = 8, h = 28, w = 28; + int64_t xDim[] = {n, g, c, d, h, w}; + int64_t xStr[] = {g * c * d * h * w, c * d * h * w, d * h * w, h * w, w, 1}; + int64_t xUi = 'x'; + int64_t alignment = 4; + checkCUDNN(cudnnBackendSetAttribute(xDesc, CUDNN_ATTR_TENSOR_DIMENSIONS, CUDNN_TYPE_INT64, 6, xDim)); + checkCUDNN(cudnnBackendSetAttribute(xDesc, CUDNN_ATTR_TENSOR_STRIDES, CUDNN_TYPE_INT64, 6, xStr)); + checkCUDNN(cudnnBackendSetAttribute(xDesc, CUDNN_ATTR_TENSOR_UNIQUE_ID, CUDNN_TYPE_INT64, 1, &xUi)); + checkCUDNN(cudnnBackendSetAttribute(xDesc, CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT, CUDNN_TYPE_INT64, 1, &alignment)); + checkCUDNN(cudnnBackendFinalize(xDesc)); + + cudnnBackendDescriptor_t wDesc; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_TENSOR_DESCRIPTOR, &wDesc)); + checkCUDNN(cudnnBackendSetAttribute(wDesc, CUDNN_ATTR_TENSOR_DATA_TYPE, CUDNN_TYPE_DATA_TYPE, 1, &dtype)); + // [g, k, c, t, r, s] = [4, 32, 16, 4, 4, 4] + int64_t k = 32, t = 4, r = 4, s = 4; + int64_t wDim[] = {g, k, c, t, r, s}; + int64_t wStr[] = {k * c * t * r * s, c * t * r * s, t * r * s, r * s, s, 1}; + int64_t wUi = 'w'; + checkCUDNN(cudnnBackendSetAttribute(wDesc, CUDNN_ATTR_TENSOR_DIMENSIONS, CUDNN_TYPE_INT64, 6, wDim)); + checkCUDNN(cudnnBackendSetAttribute(wDesc, CUDNN_ATTR_TENSOR_STRIDES, CUDNN_TYPE_INT64, 6, wStr)); + checkCUDNN(cudnnBackendSetAttribute(wDesc, CUDNN_ATTR_TENSOR_UNIQUE_ID, CUDNN_TYPE_INT64, 1, &wUi)); + checkCUDNN(cudnnBackendSetAttribute(wDesc, CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT, CUDNN_TYPE_INT64, 1, &alignment)); + checkCUDNN(cudnnBackendFinalize(wDesc)); + + cudnnBackendDescriptor_t yDesc; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_TENSOR_DESCRIPTOR, &yDesc)); + checkCUDNN(cudnnBackendSetAttribute(yDesc, CUDNN_ATTR_TENSOR_DATA_TYPE, CUDNN_TYPE_DATA_TYPE, 1, &dtype)); + //[n, g, k, o, p, q] = [32, 4, 32, 4, 25, 25] + int64_t o = 4, p = 25, q = 25; + int64_t yDim[] = {n, g, k, o, p, q}; + int64_t yStr[] = {g * k * o * p * q, k * o * p * q, o * p * q, p * q, q, 1}; + int64_t yUi = 'y'; + checkCUDNN(cudnnBackendSetAttribute(yDesc, CUDNN_ATTR_TENSOR_DIMENSIONS, CUDNN_TYPE_INT64, 6, yDim)); + checkCUDNN(cudnnBackendSetAttribute(yDesc, CUDNN_ATTR_TENSOR_STRIDES, CUDNN_TYPE_INT64, 6, yStr)); + checkCUDNN(cudnnBackendSetAttribute(yDesc, CUDNN_ATTR_TENSOR_UNIQUE_ID, CUDNN_TYPE_INT64, 1, &yUi)); + checkCUDNN(cudnnBackendSetAttribute(yDesc, CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT, CUDNN_TYPE_INT64, 1, &alignment)); + checkCUDNN(cudnnBackendFinalize(yDesc)); + + + // Create, set, and finalize a convolution operator descriptor. + + cudnnBackendDescriptor_t cDesc; + int64_t nbDims = 3; + cudnnDataType_t compType = CUDNN_DATA_FLOAT; + cudnnConvolutionMode_t mode = CUDNN_CONVOLUTION; + int64_t pad[] = {0, 0, 0}; + int64_t filterStr[] = {1, 1, 1}; + int64_t dilation[] = {1, 1, 1}; + + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR, &cDesc)); + + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS, CUDNN_TYPE_INT64, 1, &nbDims)); + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_COMP_TYPE, CUDNN_TYPE_DATA_TYPE, 1, &compType)); + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_CONV_MODE, CUDNN_TYPE_CONVOLUTION_MODE, 1, &mode)); + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS, CUDNN_TYPE_INT64, nbDims, pad)); + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_POST_PADDINGS, CUDNN_TYPE_INT64, nbDims, pad)); + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_DILATIONS, CUDNN_TYPE_INT64, nbDims, dilation)); + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES, CUDNN_TYPE_INT64, nbDims, filterStr)); + checkCUDNN(cudnnBackendFinalize(cDesc)); + + //Create, set, and finalize a convolution forward operation descriptor. + + cudnnBackendDescriptor_t fprop; + float alpha = 1.0; + float beta = 0.5; + + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR, &fprop)); + checkCUDNN(cudnnBackendSetAttribute(fprop, CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_X, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &xDesc)); + checkCUDNN(cudnnBackendSetAttribute(fprop, CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_W, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &wDesc)); + checkCUDNN(cudnnBackendSetAttribute(fprop, CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_Y, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &yDesc)); + checkCUDNN(cudnnBackendSetAttribute(fprop, CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_CONV_DESC, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &cDesc)); + checkCUDNN(cudnnBackendSetAttribute(fprop, CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_ALPHA, CUDNN_TYPE_FLOAT, 1, &alpha)); + checkCUDNN(cudnnBackendSetAttribute(fprop, CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_BETA, CUDNN_TYPE_FLOAT, 1, &beta)); + checkCUDNN(cudnnBackendFinalize(fprop)); + + //Create, set, and finalize an operation graph descriptor. + + cudnnBackendDescriptor_t op_graph; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATIONGRAPH_DESCRIPTOR, &op_graph)); + checkCUDNN(cudnnBackendSetAttribute(op_graph, CUDNN_ATTR_OPERATIONGRAPH_OPS, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &fprop)); + checkCUDNN(cudnnBackendSetAttribute(op_graph, CUDNN_ATTR_OPERATIONGRAPH_HANDLE, CUDNN_TYPE_HANDLE, 1, &handle)); + checkCUDNN(cudnnBackendFinalize(op_graph)); + + //Create, set, and finalize an engine descriptor. + + cudnnBackendDescriptor_t engine; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_ENGINE_DESCRIPTOR, &engine)); + checkCUDNN(cudnnBackendSetAttribute(engine, CUDNN_ATTR_ENGINE_OPERATION_GRAPH, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &op_graph)); + int64_t gidx = 0; + checkCUDNN(cudnnBackendSetAttribute(engine, CUDNN_ATTR_ENGINE_GLOBAL_INDEX, CUDNN_TYPE_INT64, 1, &gidx)); + checkCUDNN(cudnnBackendFinalize(engine)); + + //Create, set, and finalize an engine config descriptor. + + cudnnBackendDescriptor_t engcfg; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_ENGINECFG_DESCRIPTOR, &engcfg)); + checkCUDNN(cudnnBackendSetAttribute(engcfg, CUDNN_ATTR_ENGINECFG_ENGINE, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &engine)); + checkCUDNN(cudnnBackendFinalize(engcfg)); + + //Create, set, and finalize an execution plan descriptor. Obtain workspace size to allocate. + + cudnnBackendDescriptor_t plan; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR, &plan)); + checkCUDNN(cudnnBackendSetAttribute(plan, CUDNN_ATTR_EXECUTION_PLAN_HANDLE, CUDNN_TYPE_HANDLE, 1, &handle)); + checkCUDNN(cudnnBackendSetAttribute(plan, CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &engcfg)); + checkCUDNN(cudnnBackendFinalize(plan)); + + int64_t workspaceSize; + checkCUDNN(cudnnBackendGetAttribute(plan, CUDNN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE, CUDNN_TYPE_INT64, 1, NULL, &workspaceSize)); + + //Create, set and finalize a variant pack descriptor. + float* x_host = (float*)malloc(sizeof(float) * n * g * c * d * h * w); + float* w_host = (float*)malloc(sizeof(float) * g * k * c * t * r * s); + float* y_host = (float*)malloc(sizeof(float) * n * g * k * o * p * q); + for (int i = 0; i < n * g * c * d * h * w; i++) { + x_host[i] = 0.1 * i; + w_host[i] = 0.2 * i; + y_host[i] = 0.0; + } + + float *xData, *wData, *yData; + checkCUDA(cudaMalloc((void**)&xData, sizeof(float) * n * g * c * d * h * w)); + checkCUDA(cudaMalloc((void**)&wData, sizeof(float) * g * k * c * t * r * s)); + checkCUDA(cudaMalloc((void**)&yData, sizeof(float) * n * g * k * o * p * q)); + checkCUDA(cudaMemcpy(xData, x_host, sizeof(float) * n * g * c * d * h * w, cudaMemcpyHostToDevice)); + checkCUDA(cudaMemcpy(wData, w_host, sizeof(float) * g * k * c * t * r * s, cudaMemcpyHostToDevice)); + + void *dev_ptrs[3] = {xData, wData, yData}; // device pointer + int64_t uids[3] = {'x', 'w', 'y'}; + void *workspace; + + cudnnBackendDescriptor_t varpack; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_VARIANT_PACK_DESCRIPTOR, &varpack)); + checkCUDNN(cudnnBackendSetAttribute(varpack, CUDNN_ATTR_VARIANT_PACK_DATA_POINTERS, CUDNN_TYPE_VOID_PTR, 3, dev_ptrs)); + checkCUDNN(cudnnBackendSetAttribute(varpack, CUDNN_ATTR_VARIANT_PACK_UNIQUE_IDS, CUDNN_TYPE_INT64, 3, uids)); + checkCUDNN(cudnnBackendSetAttribute(varpack, CUDNN_ATTR_VARIANT_PACK_WORKSPACE, CUDNN_TYPE_VOID_PTR, 1, &workspace)); + checkCUDNN(cudnnBackendFinalize(varpack)); + + //Execute the plan with a variant pack. + checkCUDNN(cudnnBackendExecute(handle, plan, varpack)); + checkCUDA(cudaMemcpy(y_host, yData, sizeof(float) * n * g * k * o * p * q, cudaMemcpyDeviceToHost)); + + cudaFree(xData); + cudaFree(wData); + cudaFree(yData); + + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNTrans.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNTrans.cu new file mode 100644 index 0000000..0329cbb --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNTrans.cu @@ -0,0 +1,51 @@ +#include +#include + +#define checkCUDNN(expression) \ +{ \ + cudnnStatus_t status = (expression); \ + if (status != CUDNN_STATUS_SUCCESS) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudnnGetErrorString(status) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +int main() { + cudnnHandle_t cudnn; + checkCUDNN(cudnnCreate(&cudnn)); + + // Tensor Descriptor for src and dest + cudnnTensorDescriptor_t srcDesc, destDesc; + checkCUDNN(cudnnCreateTensorDescriptor(&srcDesc)); + checkCUDNN(cudnnCreateTensorDescriptor(&destDesc)); + int srcDimA[4] = {512, 256, 1, 1}; + checkCUDNN(cudnnSetTensorNdDescriptorEx(srcDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 4, srcDimA)); + + // Filter Descriptor + cudnnFilterDescriptor_t filterDesc; + checkCUDNN(cudnnCreateFilterDescriptor(&filterDesc)); + int filterDimA[4] = {512, 1024, 1, 1}; + checkCUDNN(cudnnSetFilterNdDescriptor(filterDesc, CUDNN_DATA_FLOAT, CUDNN_TENSOR_NCHW, 4, filterDimA)); + + // Tensor Transform Descriptor + cudnnTensorTransformDescriptor_t transformDesc; + checkCUDNN(cudnnCreateTensorTransformDescriptor(&transformDesc)); + int padBeforeA[4] = {0, 0, 0, 0}; + int padAfterA[4] = {0, 0, 1, 1}; + uint32_t foldA[2] = {2, 2}; // Considering spatial dimensions only as per documentation + checkCUDNN(cudnnSetTensorTransformDescriptor(transformDesc, 4, CUDNN_TENSOR_NCHW, padBeforeA, padAfterA, foldA, CUDNN_TRANSFORM_FOLD)); + + // Initialize dest tensor descriptor based on transformation + size_t destSizeInBytes; + checkCUDNN(cudnnInitTransformDest(transformDesc, srcDesc, destDesc, &destSizeInBytes)); + + // Clean up + checkCUDNN(cudnnDestroyTensorDescriptor(srcDesc)); + checkCUDNN(cudnnDestroyTensorDescriptor(destDesc)); + checkCUDNN(cudnnDestroyFilterDescriptor(filterDesc)); + checkCUDNN(cudnnDestroyTensorTransformDescriptor(transformDesc)); + cudnnDestroy(cudnn); + + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testCublas.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testCublas.cu new file mode 100644 index 0000000..1e67765 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testCublas.cu @@ -0,0 +1,63 @@ +/** +* 30 84 138 +* 24 69 114 +* 18 54 90 +*/ +#include +#include + +int main() { + cublasHandle_t handle; + const int N = 3; + + // 初始化 cuBLAS 句柄 + cublasCreate(&handle); + + // 分配和初始化矩阵 + float a[N*N] = {1.0, 2.0, 3.0, + 4.0, 5.0, 6.0, + 7.0, 8.0, 9.0}; + float b[N*N] = {9.0, 8.0, 7.0, + 6.0, 5.0, 4.0, + 3.0, 2.0, 1.0}; + float c[N*N] = {0}; + + float *d_a, *d_b, *d_c; + cudaMalloc((void **)&d_a, sizeof(a)); + cudaMalloc((void **)&d_b, sizeof(b)); + cudaMalloc((void **)&d_c, sizeof(c)); + + // 将数据复制到设备 + cudaMemcpy(d_a, a, sizeof(a), cudaMemcpyHostToDevice); + cudaMemcpy(d_b, b, sizeof(b), cudaMemcpyHostToDevice); + + size_t workspace_size = 10 * 1024 * 1024; // 10 MB + void* workspace; + cudaMalloc(&workspace, workspace_size); + cublasSetWorkspace_v2(handle, workspace, workspace_size); + + // 执行矩阵乘法 + const float alpha = 1.0f; + const float beta = 0.0f; + cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T, N, N, N, &alpha, d_a, N, d_b, N, &beta, d_c, N); + + // 将结果复制回主机 + cudaMemcpy(c, d_c, sizeof(c), cudaMemcpyDeviceToHost); + + // 打印结果 + for (int i = 0; i < N; i++) { + for (int j = 0; j < N; j++) { + std::cout << c[i * N + j] << " "; + } + std::cout << std::endl; + } + + // 清理 + cudaFree(d_a); + cudaFree(d_b); + cudaFree(d_c); + cudaFree(workspace); + cublasDestroy(handle); + + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testCublasLt.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testCublasLt.cu new file mode 100644 index 0000000..3a578b8 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testCublasLt.cu @@ -0,0 +1,125 @@ +/** +* cuBLASLt library version:111003 +* Heuristic result: +* algorithm descriptor:11,4294967296,0,476754254757889,0,0,68,80, +* Actual size of workspace memory required:0 +* Result status:Success +* Result matrix: +* 90 114 138 +* 54 69 84 +* 18 24 30 +*/ +#include +#include +#include +#include + +// CUDA 和 cuBLASLt 错误检查宏 +#define CHECK_CUDA(expression) \ +{ \ + cudaError_t error = (expression); \ + if (error != cudaSuccess) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudaGetErrorString(error) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +#define CHECK_CUBLASLT(expression) \ +{ \ + cublasStatus_t cublasLtStatus = (expression); \ + if (cublasLtStatus != CUBLAS_STATUS_SUCCESS) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cublasLtGetStatusString(cublasLtStatus) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +int main() { + const uint64_t N = 3; + float a[N*N] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0}; + float b[N*N] = {9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0}; + float c[N*N] = {0}; + + std::cout << "cuBLASLt library version:" << cublasLtGetVersion() << std::endl; + + // 创建 cuBLASLt 句柄 + cublasLtHandle_t handle; + CHECK_CUBLASLT(cublasLtCreate(&handle)); + + // 在 GPU 上分配内存 + float *d_a, *d_b, *d_c; + CHECK_CUDA(cudaMalloc((void **)&d_a, sizeof(a))); + CHECK_CUDA(cudaMalloc((void **)&d_b, sizeof(b))); + CHECK_CUDA(cudaMalloc((void **)&d_c, sizeof(c))); + + // 将数据复制到设备 + CHECK_CUDA(cudaMemcpy(d_a, a, sizeof(a), cudaMemcpyHostToDevice)); + CHECK_CUDA(cudaMemcpy(d_b, b, sizeof(b), cudaMemcpyHostToDevice)); + + // 定义矩阵乘法操作的描述 + cublasLtMatmulDesc_t operationDesc; + cublasLtMatrixLayout_t aLayout, bLayout, cLayout; + cublasLtMatmulPreference_t preference; + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult; + + CHECK_CUBLASLT(cublasLtMatmulDescCreate(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); + cublasOperation_t transa = CUBLAS_OP_N; // 设置矩阵 A 的转置类型 + CHECK_CUBLASLT(cublasLtMatmulDescSetAttribute(operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa))); + + CHECK_CUBLASLT(cublasLtMatrixLayoutCreate(&aLayout, CUDA_R_32F, N, N - 2, N)); + CHECK_CUBLASLT(cublasLtMatrixLayoutCreate(&bLayout, CUDA_R_32F, N, N, N)); + CHECK_CUBLASLT(cublasLtMatrixLayoutCreate(&cLayout, CUDA_R_32F, N, N, N)); + CHECK_CUBLASLT(cublasLtMatrixLayoutSetAttribute(aLayout, CUBLASLT_MATRIX_LAYOUT_COLS, &N, sizeof(N))); + + size_t workspaceSize = 10 * 1024 * 1024; + CHECK_CUBLASLT(cublasLtMatmulPreferenceCreate(&preference)); + CHECK_CUBLASLT(cublasLtMatmulPreferenceSetAttribute(preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize))); + + // 获取最佳算法 + CHECK_CUBLASLT(cublasLtMatmulAlgoGetHeuristic(handle, operationDesc, aLayout, bLayout, cLayout, cLayout, preference, 1, &heuristicResult, &returnedResults)); + + if (returnedResults == 0) { + std::cerr << "No suitable algorithm found!" << std::endl; + exit(-1); + } + else { + std::cout << "Heuristic result:" << std::endl; + std::cout << "algorithm descriptor:"; + for (int i = 0; i < 8; i ++) { + std::cout << heuristicResult.algo.data[i] << ","; + } + std::cout << std::endl << "Actual size of workspace memory required:" << heuristicResult.workspaceSize << std::endl; + std::cout << "Result status:" << (heuristicResult.state == CUBLAS_STATUS_SUCCESS ? "Success" : "Failed") << std::endl; + } + + // 执行矩阵乘法 + const float alpha = 1.0f, beta = 0.0f; + CHECK_CUBLASLT(cublasLtMatmul(handle, operationDesc, &alpha, d_a, aLayout, d_b, bLayout, &beta, d_c, cLayout, d_c, cLayout, &heuristicResult.algo, nullptr, 0, 0)); + + // 将结果复制回主机 + CHECK_CUDA(cudaMemcpy(c, d_c, sizeof(c), cudaMemcpyDeviceToHost)); + + // 打印结果 + std::cout << "Result matrix:" << std::endl; + for (int i = 0; i < N; i++) { + for (int j = 0; j < N; j++) { + std::cout << c[i * N + j] << " "; + } + std::cout << std::endl; + } + + // 清理资源 + cublasLtMatrixLayoutDestroy(aLayout); + cublasLtMatrixLayoutDestroy(bLayout); + cublasLtMatrixLayoutDestroy(cLayout); + cublasLtMatmulDescDestroy(operationDesc); + cublasLtMatmulPreferenceDestroy(preference); + cudaFree(d_a); + cudaFree(d_b); + cudaFree(d_c); + cublasLtDestroy(handle); + + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testDevice.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testDevice.cu new file mode 100644 index 0000000..9a25080 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testDevice.cu @@ -0,0 +1,165 @@ +#include +#include +#include +#include +#include + +void compareDeviceProps(const cudaDeviceProp& prop1, const cudaDeviceProp& prop2) { + + for (size_t i = 0; i < sizeof(cudaDeviceProp); i++) { + if (((uint8_t*)&prop1)[i] != ((uint8_t*)&prop2)[i]) { + printf("prop1[%02lu]: %02x, prop2[%02lu]: %02x", i, ((uint8_t*)&prop1)[i], i, ((uint8_t*)&prop2)[i]); + printf(" <- Different\n"); + } + } + + if (prop1.major != prop2.major) + std::cout << "Different 'major' version: " << prop1.major << " vs " << prop2.major << std::endl; + if (prop1.minor != prop2.minor) + std::cout << "Different 'minor' version: " << prop1.minor << " vs " << prop2.minor << std::endl; + if (memcmp(prop1.name, prop2.name, sizeof(prop1.name)) != 0) + std::cout << "Different 'name': " << prop1.name << " vs " << prop2.name << std::endl; + if (prop1.totalGlobalMem != prop2.totalGlobalMem) + std::cout << "Different 'totalGlobalMem': " << prop1.totalGlobalMem << " vs " << prop2.totalGlobalMem << std::endl; + if (prop1.sharedMemPerBlock != prop2.sharedMemPerBlock) + std::cout << "Different 'sharedMemPerBlock': " << prop1.sharedMemPerBlock << " vs " << prop2.sharedMemPerBlock << std::endl; + if (prop1.regsPerBlock != prop2.regsPerBlock) + std::cout << "Different 'regsPerBlock': " << prop1.regsPerBlock << " vs " << prop2.regsPerBlock << std::endl; + if (prop1.warpSize != prop2.warpSize) + std::cout << "Different 'warpSize': " << prop1.warpSize << " vs " << prop2.warpSize << std::endl; + if (prop1.memPitch != prop2.memPitch) + std::cout << "Different 'memPitch': " << prop1.memPitch << " vs " << prop2.memPitch << std::endl; + if (prop1.maxThreadsPerBlock != prop2.maxThreadsPerBlock) + std::cout << "Different 'maxThreadsPerBlock': " << prop1.maxThreadsPerBlock << " vs " << prop2.maxThreadsPerBlock << std::endl; + if (memcmp(prop1.maxThreadsDim, prop2.maxThreadsDim, sizeof(prop1.maxThreadsDim)) != 0) + std::cout << "Different 'maxThreadsDim': " << prop1.maxThreadsDim[0] << ", " << prop1.maxThreadsDim[1] << ", " << prop1.maxThreadsDim[2] + << " vs " << prop2.maxThreadsDim[0] << ", " << prop2.maxThreadsDim[1] << ", " << prop2.maxThreadsDim[2] << std::endl; + if (memcmp(prop1.maxGridSize, prop2.maxGridSize, sizeof(prop1.maxGridSize)) != 0) + std::cout << "Different 'maxGridSize': " << prop1.maxGridSize[0] << ", " << prop1.maxGridSize[1] << ", " << prop1.maxGridSize[2] + << " vs " << prop2.maxGridSize[0] << ", " << prop2.maxGridSize[1] << ", " << prop2.maxGridSize[2] << std::endl; + if (prop1.clockRate != prop2.clockRate) + std::cout << "Different 'clockRate': " << prop1.clockRate << " vs " << prop2.clockRate << std::endl; + if (prop1.totalConstMem != prop2.totalConstMem) + std::cout << "Different 'totalConstMem': " << prop1.totalConstMem << " vs " << prop2.totalConstMem << std::endl; + if (prop1.multiProcessorCount != prop2.multiProcessorCount) + std::cout << "Different 'multiProcessorCount': " << prop1.multiProcessorCount << " vs " << prop2.multiProcessorCount << std::endl; + if (prop1.kernelExecTimeoutEnabled != prop2.kernelExecTimeoutEnabled) + std::cout << "Different 'kernelExecTimeoutEnabled': " << prop1.kernelExecTimeoutEnabled << " vs " << prop2.kernelExecTimeoutEnabled << std::endl; + if (prop1.integrated != prop2.integrated) + std::cout << "Different 'integrated': " << prop1.integrated << " vs " << prop2.integrated << std::endl; + if (prop1.canMapHostMemory != prop2.canMapHostMemory) + std::cout << "Different 'canMapHostMemory': " << prop1.canMapHostMemory << " vs " << prop2.canMapHostMemory << std::endl; + if (prop1.computeMode != prop2.computeMode) + std::cout << "Different 'computeMode': " << prop1.computeMode << " vs " << prop2.computeMode << std::endl; + if (prop1.concurrentKernels != prop2.concurrentKernels) + std::cout << "Different 'concurrentKernels': " << prop1.concurrentKernels << " vs " << prop2.concurrentKernels << std::endl; + if (prop1.ECCEnabled != prop2.ECCEnabled) + std::cout << "Different 'ECCEnabled': " << prop1.ECCEnabled << " vs " << prop2.ECCEnabled << std::endl; + if (prop1.pciBusID != prop2.pciBusID) + std::cout << "Different 'pciBusID': " << prop1.pciBusID << " vs " << prop2.pciBusID << std::endl; + if (prop1.pciDeviceID != prop2.pciDeviceID) + std::cout << "Different 'pciDeviceID': " << prop1.pciDeviceID << " vs " << prop2.pciDeviceID << std::endl; + if (prop1.pciDomainID != prop2.pciDomainID) + std::cout << "Different 'pciDomainID': " << prop1.pciDomainID << " vs " << prop2.pciDomainID << std::endl; + if (prop1.tccDriver != prop2.tccDriver) + std::cout << "Different 'tccDriver': " << prop1.tccDriver << " vs " << prop2.tccDriver << std::endl; + if (prop1.asyncEngineCount != prop2.asyncEngineCount) + std::cout << "Different 'asyncEngineCount': " << prop1.asyncEngineCount << " vs " << prop2.asyncEngineCount << std::endl; + if (prop1.unifiedAddressing != prop2.unifiedAddressing) + std::cout << "Different 'unifiedAddressing': " << prop1.unifiedAddressing << " vs " << prop2.unifiedAddressing << std::endl; + if (prop1.memoryClockRate != prop2.memoryClockRate) + std::cout << "Different 'memoryClockRate': " << prop1.memoryClockRate << " vs " << prop2.memoryClockRate << std::endl; + if (prop1.memoryBusWidth != prop2.memoryBusWidth) + std::cout << "Different 'memoryBusWidth': " << prop1.memoryBusWidth << " vs " << prop2.memoryBusWidth << std::endl; + if (prop1.l2CacheSize != prop2.l2CacheSize) + std::cout << "Different 'l2CacheSize': " << prop1.l2CacheSize << " vs " << prop2.l2CacheSize << std::endl; + if (prop1.maxThreadsPerMultiProcessor != prop2.maxThreadsPerMultiProcessor) + std::cout << "Different 'maxThreadsPerMultiProcessor': " << prop1.maxThreadsPerMultiProcessor << " vs " << prop2.maxThreadsPerMultiProcessor << std::endl; + if (prop1.streamPrioritiesSupported != prop2.streamPrioritiesSupported) + std::cout << "Different 'streamPrioritiesSupported': " << prop1.streamPrioritiesSupported << " vs " << prop2.streamPrioritiesSupported << std::endl; + if (prop1.globalL1CacheSupported != prop2.globalL1CacheSupported) + std::cout << "Different 'globalL1CacheSupported': " << prop1.globalL1CacheSupported << " vs " << prop2.globalL1CacheSupported << std::endl; + if (prop1.localL1CacheSupported != prop2.localL1CacheSupported) + std::cout << "Different 'localL1CacheSupported': " << prop1.localL1CacheSupported << " vs " << prop2.localL1CacheSupported << std::endl; + if (prop1.sharedMemPerMultiprocessor != prop2.sharedMemPerMultiprocessor) + std::cout << "Different 'sharedMemPerMultiprocessor': " << prop1.sharedMemPerMultiprocessor << " vs " << prop2.sharedMemPerMultiprocessor << std::endl; + if (prop1.reservedSharedMemPerBlock != prop2.reservedSharedMemPerBlock) + std::cout << "Different 'reservedSharedMemPerBlock': " << prop1.reservedSharedMemPerBlock << " vs " << prop2.reservedSharedMemPerBlock << std::endl; + if (prop1.hostNativeAtomicSupported != prop2.hostNativeAtomicSupported) + std::cout << "Different 'hostNativeAtomicSupported': " << prop1.hostNativeAtomicSupported << " vs " << prop2.hostNativeAtomicSupported << std::endl; + if (prop1.singleToDoublePrecisionPerfRatio != prop2.singleToDoublePrecisionPerfRatio) + std::cout << "Different 'singleToDoublePrecisionPerfRatio': " << prop1.singleToDoublePrecisionPerfRatio << " vs " << prop2.singleToDoublePrecisionPerfRatio << std::endl; + if (prop1.pageableMemoryAccess != prop2.pageableMemoryAccess) + std::cout << "Different 'pageableMemoryAccess': " << prop1.pageableMemoryAccess << " vs " << prop2.pageableMemoryAccess << std::endl; + if (prop1.pageableMemoryAccessUsesHostPageTables != prop2.pageableMemoryAccessUsesHostPageTables) + std::cout << "Different 'pageableMemoryAccessUsesHostPageTables': " << prop1.pageableMemoryAccessUsesHostPageTables << " vs " << prop2.pageableMemoryAccessUsesHostPageTables << std::endl; + if (prop1.directManagedMemAccessFromHost != prop2.directManagedMemAccessFromHost) + std::cout << "Different 'directManagedMemAccessFromHost': " << prop1.directManagedMemAccessFromHost << " vs " << prop2.directManagedMemAccessFromHost << std::endl; +} + +int main(int argc, char* argv[]) { + int gpuid = 0; + cudaGetDevice(&gpuid); + std::cout << "gpuid: " << gpuid << std::endl; + + int cnt = 0; + cudaGetDeviceCount(&cnt); + std::cout<<"cnt: "<name: %s\n", deviceProp.name); + printf("prop->totalGlobalMem: %lu\n", deviceProp.totalGlobalMem); + //printf("prop->sharedMemPerBlock: %lu\n", prop->sharedMemPerBlock); + //printf("prop->regsPerBlock: %d\n", prop->regsPerBlock); + printf("prop->warpSize: %d\n", deviceProp.warpSize); + //printf("prop->memPitch: %lu\n", prop->memPitch); + printf("prop->maxThreadsPerBlock: %d\n", deviceProp.maxThreadsPerBlock); + printf("prop->maxThreadsPerMultiProcessor: %d\n", deviceProp.maxThreadsPerMultiProcessor); + + + // // 打开文件并读取内容 + // std::ifstream inFile("monitor-2.bin", std::ios::in | std::ios::binary); + // if (!inFile) { + // std::cerr << "Failed to open file for reading." << std::endl; + // return 1; + // } + // else { + // std::cout << "File opened: monitor-2.bin" << std::endl; + // } + + // cudaDeviceProp fileDeviceProp; + // inFile.read(reinterpret_cast(&fileDeviceProp), sizeof(cudaDeviceProp)); + + // // 关闭文件 + // inFile.close(); + + // std::cout << "=====================" << std::endl; + // // 比较从文件读取的cudaDeviceProp和从CUDA API获取的cudaDeviceProp + // compareDeviceProps(deviceProp, fileDeviceProp); + // std::cout << "=====================" << std::endl; + + + int val; + cudaDeviceGetAttribute(&val, cudaDevAttrMaxThreadsPerBlock, devID); + std::cout<<"maxThreadsPerBlock: "< +#include +#include +#include + +int main(int argc, char* argv[]) { + // float time_elapsed = 0; + cudaEvent_t start,end; + cudaEventCreate(&start); + cudaEventCreateWithFlags(&end,0); + printf("start = %p, end = %p\n",start,end); + + cudaEventRecord(start,0); + int *dev_A; + cudaMalloc(&dev_A,8); + cudaEventRecord(end,0); + + cudaEventQuery(start); + cudaEventQuery(end); + + cudaEventDestroy(start); + cudaEventDestroy(end); + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testGraph.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testGraph.cu new file mode 100644 index 0000000..fe03d33 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testGraph.cu @@ -0,0 +1,79 @@ +#include +#include + +#define N 1024 * 1024 // 数据大小 +#define THREADS_PER_BLOCK 256 + +__global__ void addKernel(float* data, int n) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx < n) { + data[idx] = data[idx] + 1; + } +} + +// 检查 CUDA 错误 +void checkCudaError(cudaError_t err, const char* msg) { + if (err != cudaSuccess) { + std::cerr << msg << ": " << cudaGetErrorString(err) << std::endl; + exit(EXIT_FAILURE); + } +} + +int main() { + // 1. 初始化数据 + float* h_data = new float[N]; + for (int i = 0; i < N; ++i) { + h_data[i] = static_cast(i); + } + + float* d_data; + checkCudaError(cudaMalloc(&d_data, N * sizeof(float)), "Failed to allocate device memory"); + checkCudaError(cudaMemcpy(d_data, h_data, N * sizeof(float), cudaMemcpyHostToDevice), "Failed to copy data to device"); + + // 2. 创建 CUDA Stream + cudaStream_t stream; + checkCudaError(cudaStreamCreate(&stream), "Failed to create CUDA stream"); + + // 3. CUDA Graph 相关变量 + cudaGraph_t graph; + cudaGraphExec_t graphExec; + + // 4. 开始捕获 CUDA Graph + checkCudaError(cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal), "Failed to begin CUDA Graph capture"); + + // 计算内核配置 + int blocks = (N + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + addKernel<<>>(d_data, N); + + // 停止捕获并生成 CUDA Graph + checkCudaError(cudaStreamEndCapture(stream, &graph), "Failed to end CUDA Graph capture"); + + // 5. 实例化 CUDA Graph + checkCudaError(cudaGraphInstantiate(&graphExec, graph, nullptr, nullptr, 0), "Failed to instantiate CUDA Graph"); + + // 6. 重复执行 CUDA Graph + const int iterations = 5; + for (int i = 0; i < iterations; ++i) { + checkCudaError(cudaGraphLaunch(graphExec, stream), "Failed to launch CUDA Graph"); + checkCudaError(cudaStreamSynchronize(stream), "Failed to synchronize stream"); + } + + cudaDeviceSynchronize(); + + // 7. 将结果从设备复制回主机 + checkCudaError(cudaMemcpy(h_data, d_data, N * sizeof(float), cudaMemcpyDeviceToHost), "Failed to copy data back to host"); + + // 8. 检查结果 + for (int i = 0; i < 5; ++i) { + std::cout << "h_data[" << i << "] = " << h_data[i] << std::endl; + } + + // 9. 释放资源 + cudaGraphExecDestroy(graphExec); + cudaGraphDestroy(graph); + cudaStreamDestroy(stream); + cudaFree(d_data); + delete[] h_data; + + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testHighThread.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testHighThread.cu new file mode 100644 index 0000000..f787014 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testHighThread.cu @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define checkCUDA(expression) \ +{ \ + cudaError_t error = (expression); \ + if (error != cudaSuccess) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudaGetErrorString(error) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +thread_local int t_tid = 0; + +inline int tid() +{ + if (t_tid == 0) + { + t_tid = static_cast(::syscall(SYS_gettid)); + } + return t_tid; +} + +// Function to be executed by each thread +void cudaTask(int priority) { + int device; + + // if (priority == 0) { + // sleep(5); + // } + + + // Get current device + cudaGetDevice(&device); + // printf("Thread %d, Current CUDA device: %d\n", tid(), device); + + // Set device to 0 + checkCUDA(cudaSetDevice(0)); + + // Allocate memory on device + float* dev_A; + size_t size = 256 * 1024 * 1024; // 256 MB + checkCUDA(cudaMalloc((void**)&dev_A, size)); + // printf("Thread %d, Allocated %.2f MB on device %d (%p)\n", tid(), size / (1024.0 * 1024.0), device, dev_A); + + // Example host data initialization + float* host_A = new float[size / sizeof(float)]; + for (size_t i = 0; i < size / sizeof(float); i++) { + host_A[i] = float(i); + } + float* host_B = new float[8]; + + // Asynchronous copy of data from host to device + for (size_t i = 0; i < 100; i++) { + checkCUDA(cudaMemcpyAsync(dev_A, host_A, size, cudaMemcpyHostToDevice)); + checkCUDA(cudaMemcpyAsync(host_B, dev_A, 8 * sizeof(float), cudaMemcpyDeviceToHost)); + } + // printf("Thread %d, Copied data from host to device and back\n", tid()); + // for (size_t i = 0; i < 8; i++) { + // printf("%.2f%c", host_B[i], (i < 7) ? ' ' : '\n'); + // } + checkCUDA(cudaDeviceSynchronize()); + + // Clean up + checkCUDA(cudaFree(dev_A)); + + // Change to device 1 + checkCUDA(cudaSetDevice(1)); + + + delete[] host_A; + delete[] host_B; +} + +int main() { + // Initialize CUDA device + cudaSetDevice(1); + + const int numThreads = 16; + std::vector threads; + + struct timeval timestart; + struct timeval timeend; + gettimeofday(×tart, NULL); + + // Create two threads using STL + for (int i = 0; i < numThreads; ++i) { + threads.emplace_back(cudaTask, i); + } + + // Wait for threads to finish + for (std::thread& t : threads) { + t.join(); + } + + // cudaTask(1); + + gettimeofday(&timeend, NULL); + long diff = 1000000 * (timeend.tv_sec - timestart.tv_sec) + + timeend.tv_usec - timestart.tv_usec; + double msTotalTime = 1.0f * diff / 1000.0; + printf("%lf\n", msTotalTime); + + // std::cout << "All threads have finished." << std::endl; + + return 0; +} + diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testKernelLaunch.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testKernelLaunch.cu new file mode 100644 index 0000000..e906606 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testKernelLaunch.cu @@ -0,0 +1,87 @@ +#include +#include +#include +#include +//#define N 1024 +//#define N 512 + + +__global__ void add(const int *A, const int *B, int *C, size_t n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + C[i] = A[i] + B[i]; +} + +__global__ void sub(const int *A, const int *B, int *C, size_t n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + C[i] = A[i] + B[i]; +} + +int main(int argc, char* argv[]) { + // cudaDeviceSynchronize(); + cudaSetDevice(0); + int testSize = 4096; + const char* logDir; + if (argc == 3) { + testSize = atoi(argv[2]); + logDir = argv[1]; + } + if (argc != 3 && argc != 1){ + printf("Usage: %s [path to the log file] [testSize]\n", argv[0]); + return 0; + } + int* A_arr = (int*)malloc(testSize); + int* B_arr = (int*)malloc(testSize); + int* C_arr = (int*)malloc(testSize); + int num = testSize / sizeof(int); + struct timeval timestart; + struct timeval timeend; + gettimeofday(×tart, NULL); + for (size_t i = 0; i < num; i++) + A_arr[i] = i, B_arr[i] = 1, C_arr[i] = 0; + + int *dev_A, *dev_B, *dev_C; + assert(cudaSuccess == cudaMalloc(&dev_A, testSize)); + assert(cudaSuccess == cudaMalloc(&dev_B, testSize)); + assert(cudaSuccess == cudaMalloc(&dev_C, testSize)); + assert(cudaSuccess == + cudaMemcpy(dev_A, A_arr, testSize, cudaMemcpyHostToDevice)); + assert(cudaSuccess == + cudaMemcpy(dev_B, B_arr, testSize, cudaMemcpyHostToDevice)); + assert(cudaSuccess == + cudaMemcpy(dev_C, C_arr, testSize, cudaMemcpyHostToDevice)); + std::cout << "address of dev_A = " << dev_A << ", dev_B = " << dev_B + << ", dev_C = " << dev_C << std::endl; + + size_t blockDim_x = 512; + //size_t blockDim_x = 256; + size_t nr_block = (num + blockDim_x - 1) / blockDim_x; + std::cout << "nr_block = " << nr_block << ", blockDim_x = " << blockDim_x + << std::endl; + add<<>>(dev_A, dev_B, dev_C, num); + std::cout << "before cudaMemcpy, address of dev_C = " << dev_C << std::endl; + cudaDeviceSynchronize(); + assert(cudaSuccess == + cudaMemcpy(C_arr, dev_C, testSize, cudaMemcpyDeviceToHost)); + + cudaFree(dev_A); + cudaFree(dev_B); + cudaFree(dev_C); + + std::cout << "C = {"; + for (size_t i = num - 10; i < num; i++) + std::cout << C_arr[i] << ", "; + std::cout << "}" << std::endl; + gettimeofday(&timeend, NULL); + long diff = 1000000 * (timeend.tv_sec - timestart.tv_sec) + + timeend.tv_usec - timestart.tv_usec; + double msTotalTime = 1.0f * diff / 1000.0; + printf("Total time = %lf ms\n", msTotalTime); + if (argc == 3) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << msTotalTime << std::endl; + outFile.close(); + } + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testKernelwithStruct.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testKernelwithStruct.cu new file mode 100644 index 0000000..7667af8 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testKernelwithStruct.cu @@ -0,0 +1,82 @@ +#include +#include +#include +#include + +typedef struct { + float a; + float b; + float *z; // pointer to device memory + float *w; // another pointer to device memory +} NestedStruct; + +typedef struct { + int x; + float y; + NestedStruct nested; + float *host_data; // pointer to host memory +} MyStruct; + +__global__ void myKernel(MyStruct myStruct, int offset) { + int idx = threadIdx.x + blockIdx.x * blockDim.x; + myStruct.nested.z[idx] = (myStruct.nested.z[idx] * myStruct.x * myStruct.y * myStruct.nested.a * myStruct.nested.b) + offset; + myStruct.nested.w[idx] = myStruct.nested.z[idx] + myStruct.nested.w[idx]; +} + +int main(int argc, char* argv[]) { + // 定义并初始化结构体 + MyStruct h_myStruct; + h_myStruct.x = 2; + h_myStruct.y = 3.5; + h_myStruct.nested.a = 1.1; // 初始化嵌套结构体成员 + h_myStruct.nested.b = 2.2; + + // 在设备上分配内存并初始化 + float h_z[10] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + float h_w[10] = {10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0}; + float *d_z, *d_w; + cudaMalloc((void**)&d_z, 10 * sizeof(float)); + cudaMalloc((void**)&d_w, 10 * sizeof(float)); + cudaMemcpy(d_z, h_z, 10 * sizeof(float), cudaMemcpyHostToDevice); + cudaMemcpy(d_w, h_w, 10 * sizeof(float), cudaMemcpyHostToDevice); + + // 将设备指针赋值给嵌套结构体 + h_myStruct.nested.z = d_z; + h_myStruct.nested.w = d_w; + + // 在主机上分配内存并初始化 + float h_host_data[10] = {0.0}; + h_myStruct.host_data = h_host_data; + + // 定义offset + int offset = 5; + + // 将结构体作为参数传递给 Kernel 函数，并传递 offset + myKernel<<<1, 10>>>(h_myStruct, offset); + + // 同步并检查错误 + cudaDeviceSynchronize(); + + // 将结果从设备内存复制回主机内存 + cudaMemcpy(h_z, d_z, 10 * sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(h_w, d_w, 10 * sizeof(float), cudaMemcpyDeviceToHost); + + // 打印结果 + printf("Results in z:\n"); + for (int i = 0; i < 10; ++i) { + printf("%f ", h_z[i]); + } + printf("\n"); + + printf("Results in w:\n"); + for (int i = 0; i < 10; ++i) { + printf("%f ", h_w[i]); + } + printf("\n"); + + // 释放设备内存 + cudaFree(d_z); + cudaFree(d_w); + + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testMemory.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testMemory.cu new file mode 100644 index 0000000..9a9fc1b --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testMemory.cu @@ -0,0 +1,82 @@ +/** +* freeMem: 16603348992, totalMem: 16928342016 +* B_arr addr: 0x7f06c7ffe010, data: {16843009,16843009,16843009,16843009,16843009,16843009,16843009,16843009,16843009,16843009} +* C_arr addr: 0x7f06a7ffd010, data: {134217718,134217719,134217720,134217721,134217722,134217723,134217724,134217725,134217726,134217727} +* Total time = 370.274000 ms +*/ +#include +#include +#include +#include +#include +//#define N 4096 + + +int main(int argc, char* argv[]) { + cudaDeviceSynchronize(); + int testSize = 512 * 1024 * 1024; // 4 MB 测试 RNDV + const char* logDir; + if (argc == 3) { + testSize = atoi(argv[2]); + logDir = argv[1]; + } + if (argc != 3 && argc != 1){ + printf("Usage: %s [testSize] [path to the log file]\n", argv[0]); + return 0; + } + size_t freeMem, totalMem; + cudaMemGetInfo(&freeMem, &totalMem); + printf("freeMem: %lu, totalMem: %lu\n", freeMem, totalMem); + + int* A_arr = (int*)malloc(testSize); + int* B_arr = (int*)malloc(testSize); + int* C_arr = (int*)malloc(testSize); + int num = testSize / sizeof(int); + for (size_t i = 0; i < num; i++) + A_arr[i] = i, B_arr[i] = 0, C_arr[i] = 0; + + struct timeval timestart; + struct timeval timeend; + gettimeofday(×tart, NULL); + + int *dev_A, *dev_B; + + assert(cudaSuccess == cudaMalloc(&dev_A, testSize)); + assert(cudaSuccess == cudaMalloc(&dev_B, testSize)); + assert(cudaSuccess == + cudaMemcpy(dev_A, A_arr, testSize, cudaMemcpyHostToDevice)); + assert(cudaSuccess == + cudaMemset(dev_B, 1, testSize)); + // assert(cudaSuccess == + // cudaMemcpy(dev_B, B_arr, testSize, cudaMemcpyHostToDevice)); + // cudaDeviceSynchronize(); + assert(cudaSuccess == + cudaMemcpy(B_arr, dev_B, testSize, cudaMemcpyDeviceToHost)); + assert(cudaSuccess == + cudaMemcpy(C_arr, dev_A, testSize, cudaMemcpyDeviceToHost)); + //cudaDeviceSynchronize(); + + printf("B_arr addr: %p, data: {", B_arr); + + + for (size_t i = max(num - 10, 0); i < num; i++) // 16843009 = 0000 0001 0000 0001 0000 0001 0000 0001 + printf("%d%s", B_arr[i], (i == num - 1) ? "}\n" : ","); + + printf("C_arr addr: %p, data: {", C_arr); + for (size_t i = max(num - 10, 0); i < num; i++) + printf("%d%s", C_arr[i], (i == num - 1) ? "}\n" : ","); + + cudaFree(dev_A); + cudaFree(dev_B); + gettimeofday(&timeend, NULL); + long diff = 1000000 * (timeend.tv_sec - timestart.tv_sec) + + timeend.tv_usec - timestart.tv_usec; + double msTotalTime = 1.0f * diff / 1000.0; + printf("Total time = %lf ms\n", msTotalTime); + if (argc == 3) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << msTotalTime << std::endl; + outFile.close(); + } + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testMemory2.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testMemory2.cu new file mode 100644 index 0000000..a6b7e12 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testMemory2.cu @@ -0,0 +1,41 @@ +// main.cu +#include +#include +#include +#include +#include +//#define N 4096 + + +int main(int argc, char* argv[]) { + int nelem = 1048576; + size_t bytes = nelem * sizeof(int); + + + size_t freeMem, totalMem; + cudaMemGetInfo(&freeMem, &totalMem); + printf("freeMem: %lu, totalMem: %lu\n", freeMem, totalMem); + + int *A_arr, *B_arr, *C_arr; + // testcodes: https://github.com/NVIDIA/cuda-samples/blob/e8568c417356f7e66bb9b7130d6be7e55324a519/Samples/2_Concepts_and_Techniques/inlinePTX/inlinePTX.cu#L82 + cudaMallocHost((void**)&A_arr, bytes); + //cudaMallocHost((void**)&B_arr, bytes); + cudaHostAlloc((void**)&B_arr, bytes, cudaHostAllocDefault); + // A_arr = (int*)malloc(bytes); + // B_arr = (int*)malloc(bytes); + C_arr = (int*)malloc(bytes); + + for (int i = 0; i < nelem; i++) { + A_arr[i] = i; + B_arr[i] = 2; + C_arr[i] = 0; + } + int idx = nelem - 2; + printf("A_arr addr = %p, A_arr[%d] = %d\n", A_arr, idx, A_arr[idx]); + printf("B_arr addr = %p, B_arr[%d] = %d\n", B_arr, idx, B_arr[idx]); + + + + + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclClient.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclClient.cu new file mode 100644 index 0000000..ae12cd0 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclClient.cu @@ -0,0 +1,198 @@ +#include "./include/nccl_util.h" + +int main(int argc, char *argv[]) { + int version; + NCCLCHECK(ncclGetVersion(&version)); + printf("NCCL version: %d\n", version); + + // Initialize client socket + const char* server_ip = (argc > 1) ? argv[1] : SERVER_IP; + int sock = 0; + struct sockaddr_in serv_addr; + + if ((sock = socket(AF_INET, SOCK_STREAM, 0)) < 0) { + printf("\n Socket creation error \n"); + return -1; + } + + serv_addr.sin_family = AF_INET; + serv_addr.sin_port = htons(SERVER_PORT); + + if (inet_pton(AF_INET, server_ip, &serv_addr.sin_addr) <= 0) { + printf("\nInvalid address/ Address not supported \n"); + return -1; + } + + if (connect(sock, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + printf("\nConnection Failed \n"); + return -1; + } + + // Receiving NCCL unique ID from server + ncclUniqueId id; + int nRanks; + int baseRank; + ssize_t bytesRead = read(sock, &id, sizeof(id)); + if (bytesRead < 0) { + printf("\nError reading NCCL unique ID\n"); + return -1; + } + bytesRead = read(sock, &nRanks, sizeof(nRanks)); + if (bytesRead < 0) { + printf("\nError reading nRanks\n"); + return -1; + } + bytesRead = read(sock, &baseRank, sizeof(baseRank)); + if (bytesRead < 0) { + printf("\nError reading baseRank\n"); + return -1; + } + close(sock); + printf("Received NCCL unique ID, nRanks=%d, baseRank=%d\n", nRanks, baseRank); + + // Initialize CUDA resources + float **sendbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **recvbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **allgatherbuff = (float **)malloc(BUFFER_SIZE * nRanks * sizeof(float *)); + float **reducescatterbuff = (float **)malloc(BUFFER_SIZE / nRanks * sizeof(float *)); + float **reducebuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **broadcastbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **sendrecvbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + cudaStream_t *s = (cudaStream_t *)malloc(sizeof(cudaStream_t) * GPUS_NUM); + float* tmpbuff = (float*)malloc(BUFFER_SIZE * sizeof(float)); + for (int i = 0; i < BUFFER_SIZE; i++) tmpbuff[i] = 2; + + for (int devID = 0; devID < GPUS_NUM; ++devID) { + CUDACHECK(cudaSetDevice(devID)); + CUDACHECK(cudaStreamCreate(s + devID)); + CUDACHECK(cudaMalloc(sendbuff + devID, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(recvbuff + devID, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(allgatherbuff + devID, BUFFER_SIZE * nRanks * sizeof(float))); + CUDACHECK(cudaMalloc(reducescatterbuff + devID, BUFFER_SIZE / nRanks * sizeof(float))); + CUDACHECK(cudaMalloc(reducebuff + devID, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(broadcastbuff + devID, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(sendrecvbuff + devID, BUFFER_SIZE * sizeof(float))); + + CUDACHECK(cudaMemcpy(sendbuff[devID], tmpbuff, BUFFER_SIZE * sizeof(float), cudaMemcpyHostToDevice)); + + CUDACHECK(cudaMemset(recvbuff[devID], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(allgatherbuff[devID], 0, BUFFER_SIZE * nRanks * sizeof(float))); + CUDACHECK(cudaMemset(reducescatterbuff[devID], 0, BUFFER_SIZE / nRanks * sizeof(float))); + CUDACHECK(cudaMemset(reducebuff[devID], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(broadcastbuff[devID], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(sendrecvbuff[devID], 0, BUFFER_SIZE * sizeof(float))); + } + printf("Init CUDA resources\n"); + printf("Before operations:\n"); + PrintBuffer((void**)sendbuff); + + // Initializing NCCL + ncclComm_t comms[GPUS_NUM]; + ncclConfig_t config = NCCL_CONFIG_INITIALIZER; + config.netName = "Socket"; + config.blocking = 1; + NCCLCHECK(ncclGroupStart()); + for (int devID = 0; devID < GPUS_NUM; devID++) { + CUDACHECK(cudaSetDevice(devID)); +#ifdef USE_CONFIG + NCCLCHECK(ncclCommInitRankConfig(comms + devID, nRanks, id, baseRank + devID, &config)); +#else + NCCLCHECK(ncclCommInitRank(comms + devID, nRanks, id, baseRank + devID)); +#endif + + } + NCCLCHECK(ncclGroupEnd()); + + for (int devID = 0; devID < GPUS_NUM; devID++) { + int count, device, userRank; + NCCLCHECK(ncclCommCount(comms[devID], &count)); + NCCLCHECK(ncclCommCuDevice(comms[devID], &device)); + NCCLCHECK(ncclCommUserRank(comms[devID], &userRank)); + printf("comm: %p, CUDADevice: %d, userRank: %d, nRanks: %d\n", comms[devID], device, userRank, count); + } + + // NCCL communication + NCCLCHECK(ncclGroupStart()); + for (int devID = 0; devID < GPUS_NUM; devID++) { + // AllReduce + NCCLCHECK(ncclAllReduce((const void *)sendbuff[devID], (void *)recvbuff[devID], + BUFFER_SIZE, ncclFloat, ncclSum, comms[devID], s[devID])); + + // AllGather + NCCLCHECK(ncclAllGather((const void *)sendbuff[devID], (void *)allgatherbuff[devID], + BUFFER_SIZE, ncclFloat, comms[devID], s[devID])); + + // ReduceScatter + NCCLCHECK(ncclReduceScatter((const void *)sendbuff[devID], (void *)reducescatterbuff[devID], + BUFFER_SIZE / nRanks, ncclFloat, ncclSum, comms[devID], s[devID])); + + // Reduce + NCCLCHECK(ncclReduce((const void *)sendbuff[devID], (void *)reducebuff[devID], + BUFFER_SIZE, ncclFloat, ncclSum, 0, comms[devID], s[devID])); + + // Broadcast + NCCLCHECK(ncclBroadcast((const void *)sendbuff[0], (void *)broadcastbuff[devID], + BUFFER_SIZE, ncclFloat, 0, comms[devID], s[devID])); + } + NCCLCHECK(ncclGroupEnd()); + + // Send and Recv (must be grouped together) + NCCLCHECK(ncclGroupStart()); + for (int devID = 0; devID < GPUS_NUM; devID++) { + if (devID == 0) { + // dev #0 (client, rank2) sends to dev rank0 (server) + NCCLCHECK(ncclSend((const void *)sendbuff[devID], BUFFER_SIZE, ncclFloat, 0, comms[devID], s[devID])); + } + else if (devID == 1) { + // dev #1 (client, rank3) receives from dev rank1 (server) + NCCLCHECK(ncclRecv((void *)sendrecvbuff[devID], BUFFER_SIZE, ncclFloat, 1, comms[devID], s[devID])); + } + } + NCCLCHECK(ncclGroupEnd()); + + // Synchronizing on CUDA stream + for (int i = 0; i < GPUS_NUM; i++) { + CUDACHECK(cudaSetDevice(i)); + CUDACHECK(cudaStreamSynchronize(s[i])); + ncclResult_t ncclAsyncErr; + NCCLCHECK(ncclCommGetAsyncError(comms[i], &ncclAsyncErr)); + if (ncclAsyncErr != ncclSuccess) { + // An asynchronous error happened. Stop the operation and destroy + // the communicator + NCCLCHECK(ncclCommAbort(comms[i])); + } + } + printf("After operations:\n"); + printf("AllReduce result:\n"); + PrintBuffer((void**)recvbuff); + printf("AllGather result:\n"); + PrintBuffer((void**)allgatherbuff, BUFFER_SIZE * nRanks); + printf("ReduceScatter result:\n"); + PrintBuffer((void**)reducescatterbuff, BUFFER_SIZE / nRanks); + printf("Reduce result:\n"); + PrintBuffer((void**)reducebuff); + printf("Broadcast result:\n"); + PrintBuffer((void**)broadcastbuff); + printf("SendRecv result:\n"); + PrintBuffer((void**)sendrecvbuff); + + // Freeing device memory + for (int i = 0; i < GPUS_NUM; i++) { + CUDACHECK(cudaSetDevice(i)); + CUDACHECK(cudaFree(sendbuff[i])); + CUDACHECK(cudaFree(recvbuff[i])); + CUDACHECK(cudaFree(allgatherbuff[i])); + CUDACHECK(cudaFree(reducescatterbuff[i])); + CUDACHECK(cudaFree(reducebuff[i])); + CUDACHECK(cudaFree(broadcastbuff[i])); + CUDACHECK(cudaFree(sendrecvbuff[i])); + } + + // Finalizing NCCL + for (int devID = 0; devID < GPUS_NUM; devID++) { + NCCLCHECK(ncclCommDestroy(comms[devID])); + } + + printf("Client completed successfully\n"); + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclLocal.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclLocal.cu new file mode 100644 index 0000000..9c9a5c9 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclLocal.cu @@ -0,0 +1,147 @@ +#include "./include/nccl_util.h" + +int main(int argc, char *argv[]) { + + // managing 4 devices + int nDev = GPUS_NUM; + int devs[GPUS_NUM]; + for (int i = 0; i < nDev; ++i) devs[i] = i; + + ncclComm_t comms[GPUS_NUM]; + + // allocating and initializing device buffers + float **sendbuff = (float **)malloc(nDev * sizeof(float *)); + float **recvbuff = (float **)malloc(nDev * sizeof(float *)); + float **recvbuff2 = (float **)malloc(nDev * sizeof(float *)); + void *sendRegHandle[nDev]; + void *recvRegHandle[nDev]; + cudaStream_t *s = (cudaStream_t *)malloc(sizeof(cudaStream_t) * nDev); + float* tmpbuff = (float*)malloc(BUFFER_SIZE * sizeof(float)); + for (int i = 0; i < BUFFER_SIZE; i++) tmpbuff[i] = 1; + + for (int i = 0; i < nDev; ++i) { + CUDACHECK(cudaSetDevice(i)); + + NCCLCHECK(ncclMemAlloc((void **)sendbuff + i, BUFFER_SIZE * sizeof(float))); + NCCLCHECK(ncclMemAlloc((void **)recvbuff + i, BUFFER_SIZE * sizeof(float))); + + // CUDACHECK(cudaMalloc((void**)sendbuff + i, size * sizeof(float))); + NCCLCHECK(ncclMemAlloc((void **)recvbuff2 + i, BUFFER_SIZE * sizeof(float))); + + CUDACHECK(cudaMemcpy(sendbuff[i], tmpbuff, BUFFER_SIZE * sizeof(float), cudaMemcpyHostToDevice)); + + CUDACHECK(cudaMemset(recvbuff[i], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(recvbuff2[i], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaStreamCreate(s + i)); + } + + // initializing NCCL and preparing custom operation + NCCLCHECK(ncclCommInitAll(comms, nDev, devs)); + float scalar_1 = 2.0f, scalar_2 = 3.0f; + float *d_scalar; + // CUDACHECK(cudaMalloc(&d_scalar, sizeof(float))); + NCCLCHECK(ncclMemAlloc((void **)&d_scalar, sizeof(float))); + CUDACHECK( + cudaMemcpy(d_scalar, &scalar_2, sizeof(float), cudaMemcpyHostToDevice)); + ncclRedOp_t customOp_1[nDev], customOp_2[nDev]; + for (int i = 0; i < nDev; ++i) { + NCCLCHECK(ncclCommRegister(comms[i], sendbuff[i], BUFFER_SIZE * sizeof(float), + sendRegHandle + i)); + NCCLCHECK(ncclCommRegister(comms[i], recvbuff[i], BUFFER_SIZE * sizeof(float), + recvRegHandle + i)); + NCCLCHECK(ncclRedOpCreatePreMulSum(customOp_1 + i, &scalar_1, ncclFloat, + ncclScalarHostImmediate, comms[i])); + NCCLCHECK(ncclRedOpCreatePreMulSum(customOp_2 + i, d_scalar, ncclFloat, + ncclScalarDevice, comms[i])); + printf("CustomOp_1: %u, CustomOp_2: %u\n", customOp_1[i], customOp_2[i]); + } + + printf("Before AllGather:\n"); + PrintBuffer((void**)sendbuff); + + // calling NCCL communication API. Group API is required when using + // multiple devices per thread + NCCLCHECK(ncclGroupStart()); + for (int i = 0; i < nDev; ++i) { + NCCLCHECK(ncclAllReduce((const void *)sendbuff[i], (void *)recvbuff[i], + BUFFER_SIZE, ncclFloat, customOp_1[i], comms[i], s[i])); + } + NCCLCHECK(ncclGroupEnd()); + + // synchronizing on CUDA streams to wait for completion of NCCL operation + for (int i = 0; i < nDev; ++i) { + CUDACHECK(cudaSetDevice(i)); + CUDACHECK(cudaStreamSynchronize(s[i])); + } + +// NCCLCHECK(ncclGroupStart()); +// for (int i = 0; i < nDev; ++i) { +// NCCLCHECK(ncclAllReduce((const void *)sendbuff[i], (void *)recvbuff2[i], +// size, ncclFloat, customOp_2[i], comms[i], s[i])); +// } +// NCCLCHECK(ncclGroupEnd()); + +// // synchronizing on CUDA streams to wait for completion of NCCL operation +// for (int i = 0; i < nDev; ++i) { +// CUDACHECK(cudaSetDevice(i)); +// CUDACHECK(cudaStreamSynchronize(s[i])); +// } + + printf("AllGather result:\n"); + PrintBuffer((void**)recvbuff); + + // free device buffers + for (int i = 0; i < nDev; ++i) { + CUDACHECK(cudaSetDevice(i)); + NCCLCHECK(ncclCommDeregister(comms[i], sendRegHandle[i])); + NCCLCHECK(ncclCommDeregister(comms[i], recvRegHandle[i])); + NCCLCHECK(ncclMemFree(sendbuff[i])); + NCCLCHECK(ncclMemFree(recvbuff[i])); + NCCLCHECK(ncclMemFree(recvbuff2[i])); + // CUDACHECK(cudaFree(sendbuff[i])); + // CUDACHECK(cudaFree(recvbuff2[i])); + } + + + // finalizing NCCL + for (int i = 0; i < nDev; ++i) { + ncclRedOpDestroy(customOp_1[i], comms[i]); + ncclRedOpDestroy(customOp_2[i], comms[i]); + } + + for (int i =0; i < nDev; ++i) { + // Finalize the NCCL communicator + NCCLCHECK(ncclCommFinalize(comms[i])); + + // Check for communicator state + ncclResult_t asyncErr; + do { + NCCLCHECK(ncclCommGetAsyncError(comms[i], &asyncErr)); + } while (asyncErr == ncclInProgress); + + if (asyncErr == ncclSuccess) { + printf("Communicator successfully finalized.\n"); + NCCLCHECK(ncclCommDestroy(comms[i])); + } else { + printf("Failed to finalize communicator.\n"); + } + } + + // ncclUniqueId commId; + // NCCLCHECK(ncclGetUniqueId(&commId)); + // ncclConfig_t config = NCCL_CONFIG_INITIALIZER; + // config.netName = "Socket"; + // ncclComm_t newcomm[nDev]; + // NCCLCHECK(ncclGroupStart()); + // for (int i = 0; i < nDev; ++i) { + // NCCLCHECK(ncclCommInitRankConfig(newcomm + i, nDev, commId, i, &config)); + // } + // NCCLCHECK(ncclGroupEnd()); + + // for (int i = 0; i < nDev; ++i) { + // NCCLCHECK(ncclCommDestroy(newcomm[i])); + // } + + printf("Success \n"); + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclServer.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclServer.cu new file mode 100644 index 0000000..d47c7e9 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclServer.cu @@ -0,0 +1,216 @@ +#include "./include/nccl_util.h" + +int main(int argc, char *argv[]) { + int version; + NCCLCHECK(ncclGetVersion(&version)); + printf("NCCL version: %d\n", version); + + const char* server_ip = (argc > 1) ? argv[1] : SERVER_IP; + int nRanks = GPUS_NUM * (CLIENTS_NUM + 1); // each node has same number of GPUs + + // Initialize server socket + int server_fd, new_socket; + struct sockaddr_in address; + int opt = 1; + int addrlen = sizeof(address); + + if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) { + perror("socket failed"); + exit(EXIT_FAILURE); + } + + if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, &opt, + sizeof(opt))) { + perror("setsockopt"); + close(server_fd); + exit(EXIT_FAILURE); + } + + address.sin_family = AF_INET; + address.sin_port = htons(SERVER_PORT); + if (inet_pton(AF_INET, server_ip, &address.sin_addr) <= 0) { + printf("\nInvalid address/ Address not supported \n"); + return -1; + } + + if (bind(server_fd, (struct sockaddr *)&address, sizeof(address)) < 0) { + perror("bind failed"); + close(server_fd); + exit(EXIT_FAILURE); + } + + if (listen(server_fd, 3) < 0) { + perror("listen"); + close(server_fd); + exit(EXIT_FAILURE); + } + printf("Server listening on port %s:%d\n", server_ip, SERVER_PORT); + + // Generating NCCL unique ID + ncclUniqueId id; + ncclGetUniqueId(&id); + + // Accepting connections from clients + for (int i = 0; i < CLIENTS_NUM; ++i) { + int clientBaseRank = GPUS_NUM + i * GPUS_NUM; //start behind server + if ((new_socket = accept(server_fd, (struct sockaddr *)&address, + (socklen_t *)&addrlen)) < 0) { + perror("accept"); + close(server_fd); + exit(EXIT_FAILURE); + } + // Sending NCCL unique ID, nRanks, and its base rank to clients + send(new_socket, &id, sizeof(id), 0); + send(new_socket, &nRanks, sizeof(nRanks), 0); + send(new_socket, &clientBaseRank, sizeof(clientBaseRank), 0); + close(new_socket); + } + printf("NCCL unique ID, nRanks and base rank sent to clients\n"); + + float **sendbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **recvbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **allgatherbuff = (float **)malloc(BUFFER_SIZE * nRanks * sizeof(float *)); + float **reducescatterbuff = (float **)malloc(BUFFER_SIZE / nRanks * sizeof(float *)); + float **reducebuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **broadcastbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **sendrecvbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + cudaStream_t *s = (cudaStream_t *)malloc(sizeof(cudaStream_t) * GPUS_NUM); + // Initialize CUDA resources + float* tmpbuff = (float*)malloc(BUFFER_SIZE * sizeof(float)); + for (int i = 0; i < BUFFER_SIZE; i++) tmpbuff[i] = 1; + + for (int I = 0; I < GPUS_NUM; ++I) { + CUDACHECK(cudaSetDevice(I)); + CUDACHECK(cudaStreamCreate(s + I)); + CUDACHECK(cudaMalloc(sendbuff + I, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(recvbuff + I, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(allgatherbuff + I, BUFFER_SIZE * nRanks * sizeof(float))); + CUDACHECK(cudaMalloc(reducescatterbuff + I, BUFFER_SIZE / nRanks * sizeof(float))); + CUDACHECK(cudaMalloc(reducebuff + I, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(broadcastbuff + I, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(sendrecvbuff + I, BUFFER_SIZE * sizeof(float))); + + CUDACHECK(cudaMemcpy(sendbuff[I], tmpbuff, BUFFER_SIZE * sizeof(float), cudaMemcpyHostToDevice)); + + CUDACHECK(cudaMemset(recvbuff[I], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(allgatherbuff[I], 0, BUFFER_SIZE * nRanks * sizeof(float))); + CUDACHECK(cudaMemset(reducescatterbuff[I], 0, BUFFER_SIZE / nRanks * sizeof(float))); + CUDACHECK(cudaMemset(reducebuff[I], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(broadcastbuff[I], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(sendrecvbuff[I], 0, BUFFER_SIZE * sizeof(float))); + } + printf("Init CUDA resources\n"); + printf("Before operations:\n"); + PrintBuffer((void**)sendbuff); + + // Initializing NCCL + ncclConfig_t config = NCCL_CONFIG_INITIALIZER; + config.netName = "Socket"; + ncclComm_t comms[GPUS_NUM]; + NCCLCHECK(ncclGroupStart()); + for (int devID = 0; devID < GPUS_NUM; devID++) { + CUDACHECK(cudaSetDevice(devID)); +#ifdef USE_CONFIG + NCCLCHECK(ncclCommInitRankConfig(comms + devID, nRanks, id, devID, &config)); +#else + NCCLCHECK(ncclCommInitRank(comms + devID, nRanks, id, devID)); +#endif + } + NCCLCHECK(ncclGroupEnd()); + + for (int devID = 0; devID < GPUS_NUM; devID++) { + int count, device, userRank; + NCCLCHECK(ncclCommCount(comms[devID], &count)); + NCCLCHECK(ncclCommCuDevice(comms[devID], &device)); + NCCLCHECK(ncclCommUserRank(comms[devID], &userRank)); + printf("comm: %p, CUDADevice: %d, userRank: %d, nRanks: %d\n", comms[devID], device, userRank, count); + } + + // NCCL communication + NCCLCHECK(ncclGroupStart()); + for (int devID = 0; devID < GPUS_NUM; devID++) { + // AllReduce + NCCLCHECK(ncclAllReduce((const void *)sendbuff[devID], (void *)recvbuff[devID], + BUFFER_SIZE, ncclFloat, ncclSum, comms[devID], s[devID])); + + // AllGather + NCCLCHECK(ncclAllGather((const void *)sendbuff[devID], (void *)allgatherbuff[devID], + BUFFER_SIZE, ncclFloat, comms[devID], s[devID])); + + // ReduceScatter + NCCLCHECK(ncclReduceScatter((const void *)sendbuff[devID], (void *)reducescatterbuff[devID], + BUFFER_SIZE / nRanks, ncclFloat, ncclSum, comms[devID], s[devID])); + + // Reduce + NCCLCHECK(ncclReduce((const void *)sendbuff[devID], (void *)reducebuff[devID], + BUFFER_SIZE, ncclFloat, ncclSum, 0, comms[devID], s[devID])); + + // Broadcast + NCCLCHECK(ncclBroadcast((const void *)sendbuff[0], (void *)broadcastbuff[devID], + BUFFER_SIZE, ncclFloat, 0, comms[devID], s[devID])); + + // Send and Recv + + } + NCCLCHECK(ncclGroupEnd()); + + // Send and Recv + NCCLCHECK(ncclGroupStart()); + for (int devID = 0; devID < GPUS_NUM; devID++) { + if (devID == 1) { + // dev #1 (server, rank1) sends to dev rank3 (client) + NCCLCHECK(ncclSend((const void *)sendbuff[devID], BUFFER_SIZE, ncclFloat, 3, comms[devID], s[devID])); + } + else if (devID == 0) { + // dev #0 (server, rank0) receives from dev rank2 (client) + NCCLCHECK(ncclRecv((void *)sendrecvbuff[devID], BUFFER_SIZE, ncclFloat, 2, comms[devID], s[devID])); + } + } + NCCLCHECK(ncclGroupEnd()); + + // Synchronizing on CUDA stream + for (int i = 0; i < GPUS_NUM; i++) { + CUDACHECK(cudaSetDevice(i)); + CUDACHECK(cudaStreamSynchronize(s[i])); + ncclResult_t ncclAsyncErr; + NCCLCHECK(ncclCommGetAsyncError(comms[i], &ncclAsyncErr)); + if (ncclAsyncErr != ncclSuccess) { + // An asynchronous error happened. Stop the operation and destroy + // the communicator + NCCLCHECK(ncclCommAbort(comms[i])); + } + } + printf("After operations:\n"); + printf("AllReduce result:\n"); + PrintBuffer((void**)recvbuff); + printf("AllGather result:\n"); + PrintBuffer((void**)allgatherbuff, BUFFER_SIZE * nRanks); + printf("ReduceScatter result:\n"); + PrintBuffer((void**)reducescatterbuff, BUFFER_SIZE / nRanks); + printf("Reduce result:\n"); + PrintBuffer((void**)reducebuff); + printf("Broadcast result:\n"); + PrintBuffer((void**)broadcastbuff); + printf("SendRecv result:\n"); + PrintBuffer((void**)sendrecvbuff); + + // Freeing device memory + for (int devID = 0; devID < GPUS_NUM; devID++) { + CUDACHECK(cudaSetDevice(devID)); + CUDACHECK(cudaFree(sendbuff[devID])); + CUDACHECK(cudaFree(recvbuff[devID])); + CUDACHECK(cudaFree(allgatherbuff[devID])); + CUDACHECK(cudaFree(reducescatterbuff[devID])); + CUDACHECK(cudaFree(reducebuff[devID])); + CUDACHECK(cudaFree(broadcastbuff[devID])); + } + + // Finalizing NCCL + for (int devID = 0; devID < GPUS_NUM; devID++) { + ncclCommDestroy(comms[devID]); + } + + printf("Server completed successfully\n"); + close(server_fd); + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testOthers.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testOthers.cu new file mode 100644 index 0000000..6e9b6d1 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testOthers.cu @@ -0,0 +1,32 @@ +#include +#include +#include +#include +#include + +#define DATA_SIZE (64ULL << 20ULL) // 64MB + +__global__ void simpleKernel(char *ptr, int sz, char val) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + for (; idx < sz; idx += (gridDim.x * blockDim.x)) { + ptr[idx] = val; + } +} + +int main() { + int blocks = 0; + int threads = 128; + cudaDeviceProp prop; + // cudaStream_t stream; + // char id = 'x'; + + cudaGetDeviceProperties(&prop, 0); + std::cout << "SM count: " << prop.multiProcessorCount << std::endl; + cudaOccupancyMaxActiveBlocksPerMultiprocessor(&blocks, simpleKernel, threads, 0); + std::cout << "Max blocks per SM: " << blocks << std::endl; + + cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(&blocks, simpleKernel, threads, 0, cudaOccupancyDisableCachingOverride); + std::cout << "Max blocks per SM (no caching): " << blocks << std::endl; + + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testStream.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testStream.cu new file mode 100644 index 0000000..e747ec4 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testStream.cu @@ -0,0 +1,68 @@ +#include +#include +#include +#include +#include +#include + + +int main() { + + int leastPriority, greatestPriority; + cudaDeviceGetStreamPriorityRange(&leastPriority, &greatestPriority); + if (leastPriority == greatestPriority) { + printf("This device does not support stream priority.\n"); + } else { + printf("Priority range: [%d, %d]\n", greatestPriority, leastPriority); + } + + // Create a CUDA stream with flags + cudaStream_t stream; + assert(cudaSuccess == cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); + + // Create a CUDA stream with priority + cudaStream_t streamWithPriority; + assert(cudaSuccess == cudaStreamCreateWithPriority(&streamWithPriority, cudaStreamNonBlocking, 1)); + + // Create an event + cudaEvent_t event; + assert(cudaSuccess == cudaEventCreate(&event)); + + printf("stream = %p, streamWithPriority = %p, event = %p\n", stream, streamWithPriority, event); + + // Record an event in the stream + // cudaEventRecord(event, stream); + + // Wait for the event to complete in another stream + assert(cudaSuccess == cudaStreamWaitEvent(streamWithPriority, event, 0)); + + // Synchronize streams + assert(cudaSuccess == cudaStreamSynchronize(stream)); + assert(cudaSuccess == cudaStreamSynchronize(streamWithPriority)); + + // Check if a stream is capturing + enum cudaStreamCaptureStatus capturing; + assert(cudaSuccess == cudaStreamIsCapturing(stream, &capturing)); + if (capturing) { + std::cout << "Stream is capturing." << std::endl; + } else { + std::cout << "Stream is not capturing." << std::endl; + } + + // Get capture information + enum cudaStreamCaptureStatus status; + unsigned long long globalThreadId; + assert(cudaSuccess == cudaStreamGetCaptureInfo(stream, &status, &globalThreadId)); + std::cout << "Capture Status: " << status << std::endl; + std::cout << "Global Thread ID: " << globalThreadId << std::endl; + + // Clean up + assert(cudaSuccess == cudaStreamDestroy(stream)); + assert(cudaSuccess == cudaStreamDestroy(streamWithPriority)); + assert(cudaSuccess == cudaEventDestroy(event)); + + return 0; +} + + +//end \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testT.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testT.cu new file mode 100644 index 0000000..e281dc2 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testT.cu @@ -0,0 +1,88 @@ +#include +#include +#include + +// 错误检查宏 +#define CHECK_CUDA(call) { \ + cudaError_t cudaStatus = call; \ + if (cudaStatus != cudaSuccess) { \ + fprintf(stderr, "CUDA Error: %s\n", cudaGetErrorString(cudaStatus)); \ + exit(-1); \ + } \ +} + +// 核函数 +__global__ void addOne(float *data, int size) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < size) { + data[index] += 1.0f; + } +} + +int main() { + const int size = 9; + float *h_data = new float[size]; // 主机端内存 + float *d_data; // 设备端内存 + const int bytes = size * sizeof(float); + + // 初始化数据 + for (int i = 0; i < size; ++i) { + h_data[i] = 1.11111111111f * (i + 1); + } + + // 在 GPU 上分配内存 + CHECK_CUDA(cudaMalloc((void **)&d_data, bytes)); + + // 异步拷贝数据到 GPU + CHECK_CUDA(cudaMemcpyAsync(d_data, h_data, bytes, cudaMemcpyHostToDevice)); + + // 确定线程块大小和网格大小 + int blockSize = 256; + int gridSize = (size + blockSize - 1) / blockSize; + + struct cudaFuncAttributes funcAttrib; + CHECK_CUDA(cudaFuncGetAttributes(&funcAttrib, addOne)); + std::cout << "Kernel Function Info:" << "\n" + << " Max Threads Per Block: " << funcAttrib.maxThreadsPerBlock << "\n" + << " Shared Memory Per Block: " << funcAttrib.sharedSizeBytes << "\n" + << " Constant Memory Per Block: " << funcAttrib.constSizeBytes << "\n" + << " Local Memory Per Block: " << funcAttrib.localSizeBytes << "\n" + << " Registers Per Block: " << funcAttrib.numRegs << "\n" + << " PTX Version: " << funcAttrib.ptxVersion << "\n" + << " Binary Version: " << funcAttrib.binaryVersion << std::endl; + + // 启动核函数 + addOne<<>>(d_data, size); + CHECK_CUDA(cudaPeekAtLastError()); // 检查核函数是否启动成功 + + // 异步拷贝结果回主机 + CHECK_CUDA(cudaMemcpyAsync(h_data, d_data, bytes, cudaMemcpyDeviceToHost)); + + // 等待 GPU 完成 + CHECK_CUDA(cudaDeviceSynchronize()); + + // 打印结果 + std::cout << "Results:" << std::endl; + for (int i = 0; i < size; ++i) { + std::cout << h_data[i] << std::endl; + } + + // 清理 + delete[] h_data; + cudaFree(d_data); + + // // 假设有一个CUDA操作，这里直接使用一个错误码模拟 + // CUresult result = CUDA_ERROR_INVALID_VALUE; // 示例错误码 + + // // 获取错误描述字符串 + // const char* errorStr; + // CUresult status = cuGetErrorString(result, &errorStr); + + // if (status == CUDA_SUCCESS) { + // std::cout << "CUDA Error: " << errorStr << std::endl; + // } else { + // std::cout << "Failed to get CUDA error string." << std::endl; + // } + + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testThread.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testThread.cu new file mode 100644 index 0000000..baa1b19 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testThread.cu @@ -0,0 +1,102 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#define checkCUDA(expression) \ +{ \ + cudaError_t error = (expression); \ + if (error != cudaSuccess) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudaGetErrorString(error) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +thread_local int t_tid = 0; + +inline int tid() +{ + if (t_tid == 0) + { + t_tid = static_cast(::syscall(SYS_gettid)); + } + return t_tid; +} + +// Function to be executed by each thread +void cudaTask(int priority) { + int device; + + // if (priority == 0) { + // sleep(5); + // } + + + // Get current device + cudaGetDevice(&device); + printf("Thread %d, Current CUDA device: %d\n", tid(), device); + + // Set device to 0 + checkCUDA(cudaSetDevice(0)); + + // Allocate memory on device + float* dev_A; + size_t size = 256 * 1024 * 1024; // 256 MB + checkCUDA(cudaMalloc((void**)&dev_A, size)); + printf("Thread %d, Allocated %.2f MB on device %d (%p)\n", tid(), size / (1024.0 * 1024.0), device, dev_A); + + // Example host data initialization + float* host_A = new float[size / sizeof(float)]; + for (size_t i = 0; i < size / sizeof(float); i++) { + host_A[i] = float(i); + } + float* host_B = new float[8]; + + // Asynchronous copy of data from host to device + checkCUDA(cudaMemcpyAsync(dev_A, host_A, size, cudaMemcpyHostToDevice)); + checkCUDA(cudaMemcpyAsync(host_B, dev_A, 8 * sizeof(float), cudaMemcpyDeviceToHost)); + printf("Thread %d, Copied data from host to device and back\n", tid()); + for (size_t i = 0; i < 8; i++) { + printf("%.2f%c", host_B[i], (i < 7) ? ' ' : '\n'); + } + + // Clean up + checkCUDA(cudaFree(dev_A)); + + // Change to device 1 + checkCUDA(cudaSetDevice(1)); + + + delete[] host_A; + delete[] host_B; +} + +int main() { + // Initialize CUDA device + cudaSetDevice(1); + + const int numThreads = 16; + std::vector threads; + + // Create two threads using STL + for (int i = 0; i < numThreads; ++i) { + threads.emplace_back(cudaTask, i); + } + + // Wait for threads to finish + for (std::thread& t : threads) { + t.join(); + } + + // cudaTask(1); + + std::cout << "All threads have finished." << std::endl; + + return 0; +} + diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testZeroCopy.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testZeroCopy.cu new file mode 100644 index 0000000..ab84f54 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testZeroCopy.cu @@ -0,0 +1,87 @@ +#include +#include +#include + +__global__ void vectorAddGPU(float *a, float *b, float *c, int N) +{ + int idx = blockIdx.x * blockDim.x + threadIdx.x; + + if (idx < N) + { + c[idx] = a[idx] + b[idx]; + } +} + +#define MEMORY_ALIGNMENT 4096 +#define ALIGN_UP(x, size) (((size_t)x + (size - 1)) & (~(size - 1))) + +int main(int argc, char **argv) +{ + int n, nelem; + unsigned int flags; + size_t bytes; + float *a, *b, *c; // Pinned memory allocated on the CPU + float *d_a, *d_b, *d_c; // Device pointers for mapped memory + float errorNorm, refNorm, ref, diff; + + + /* Allocate mapped CPU memory. */ + nelem = 1048576; + bytes = nelem * sizeof(float); + + flags = cudaHostAllocMapped; + assert(cudaSuccess == cudaHostAlloc((void **)&a, bytes, flags)); + assert(cudaSuccess == cudaHostAlloc((void **)&b, bytes, flags)); + assert(cudaSuccess == cudaHostAlloc((void **)&c, bytes, flags)); + printf("a: %p, b: %p, c: %p\n", a, b, c); + + /* Initialize the vectors. */ + for (n = 0; n < nelem; n++){ + a[n] = rand() / (float)RAND_MAX; + b[n] = rand() / (float)RAND_MAX; + } + + /* Get the device pointers for the pinned CPU memory mapped into the GPU + memory space. */ + assert(cudaSuccess == cudaHostGetDevicePointer((void **)&d_a, (void *)a, 0)); + assert(cudaSuccess == cudaHostGetDevicePointer((void **)&d_b, (void *)b, 0)); + assert(cudaSuccess == cudaHostGetDevicePointer((void **)&d_c, (void *)c, 0)); + printf("d_a: %p, d_b: %p, d_c: %p\n", d_a, d_b, d_c); + + + /* Call the GPU kernel using the CPU pointers residing in CPU mapped memory. */ + printf("> vectorAddGPU kernel will add vectors using mapped CPU memory...\n"); + dim3 block(256); + dim3 grid((unsigned int)ceil(nelem / (float)block.x)); + vectorAddGPU<<>>(d_a, d_b, d_c, nelem); + + cudaDeviceSynchronize(); + + /* Compare the results */ + printf("> Checking the results from vectorAddGPU() ...\n"); + errorNorm = 0.f; + refNorm = 0.f; + for (n = 0; n < nelem; n++){ + ref = a[n] + b[n]; + diff = c[n] - ref; + errorNorm += diff * diff; + refNorm += ref * ref; + if (n < 10) + printf("c[%d]=%f, ref=%f\n", n, c[n], ref); + } + + errorNorm = (float)sqrt((double)errorNorm); + refNorm = (float)sqrt((double)refNorm); + + + + /* Memory clean up */ + printf("> Releasing CPU memory...\n"); + assert(cudaSuccess == cudaFreeHost(a)); + assert(cudaSuccess == cudaFreeHost(b)); + assert(cudaSuccess == cudaFreeHost(c)); + + + + exit(errorNorm / refNorm < 1.e-6f ? EXIT_SUCCESS : EXIT_FAILURE); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cuBlasCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cuBlasCallback.cc new file mode 100644 index 0000000..9220bf2 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cuBlasCallback.cc @@ -0,0 +1,185 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "cuBlasHandle"; + +DEFINE_SERVER_AM_CALLBACK(cublasCreate_v2Handle) { + tool::Logging(myName, "CUBLAS_CREATE_V2\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cublasHandle_t handle = NULL; + cublasStatus_t exit_code = cublasCreate_v2(&handle); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasCreate_v2 success, handle = %p\n", handle); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, handle); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLAS_CREATE_V2); + handle = (cublasHandle_t)serverEp->GetHandleVirAddr(handle, CUBLAS_CREATE_V2); + resBuf.Push64BitPointer(handle); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasCreate_v2 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasSgemm_v2Handle) { + tool::Logging(myName, "CUBLAS_SGEMM_V2\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cublasOperation_t transa = reqBuf.Pop(); + cublasOperation_t transb = reqBuf.Pop(); + int m = reqBuf.Pop(); + int n = reqBuf.Pop(); + int k = reqBuf.Pop(); + const float* alpha = reqBuf.AssignAddr(); + const float* A = (const float*)serverEp->GetDevPtr(reqBuf.Pop()); + int lda = reqBuf.Pop(); + const float* B = (const float*)serverEp->GetDevPtr(reqBuf.Pop()); + int ldb = reqBuf.Pop(); + const float* beta = reqBuf.AssignAddr(); + float* C = (float*)serverEp->GetDevPtr(reqBuf.Pop()); + int ldc = reqBuf.Pop(); + cublasStatus_t exit_code = cublasSgemm_v2(handle, transa, transb, m, n, k, + alpha, A, lda, B, ldb, beta, C, ldc); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasSgemm_v2 success, handle=%p\n", handle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasSgemm_v2 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasSgemmStridedBatchedHandle) { + tool::Logging(myName, "CUBLAS_SGEMM_STRIDED_BATCHED\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cublasOperation_t transa = reqBuf.Pop(); + cublasOperation_t transb = reqBuf.Pop(); + int m = reqBuf.Pop(); + int n = reqBuf.Pop(); + int k = reqBuf.Pop(); + const float* alpha = reqBuf.AssignAddr(); + const float* A = (const float*)serverEp->GetDevPtr(reqBuf.Pop()); + int lda = reqBuf.Pop(); + long long int strideA = reqBuf.Pop(); + const float* B = (const float*)serverEp->GetDevPtr(reqBuf.Pop()); + int ldb = reqBuf.Pop(); + long long int strideB = reqBuf.Pop(); + const float* beta = reqBuf.AssignAddr(); + float* C = (float*)serverEp->GetDevPtr(reqBuf.Pop()); + int ldc = reqBuf.Pop(); + long long int strideC = reqBuf.Pop(); + int batchCount = reqBuf.Pop(); + cublasStatus_t exit_code = cublasSgemmStridedBatched(handle, transa, transb, m, n, k, + alpha, A, lda, strideA, B, ldb, strideB, beta, + C, ldc, strideC, batchCount); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasSgemmStridedBatched success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasSgemmStridedBatched failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasDestroy_v2Handle) { + tool::Logging(myName, "CUBLAS_DESTROY_V2\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop(), true); + cublasStatus_t exit_code = cublasDestroy_v2(handle); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasDestroy_v2 success, handle = %p\n", handle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasDestroy_v2 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasSetStream_v2Handle) { + tool::Logging(myName, "CUBLAS_SET_STREAM_V2\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + cublasStatus_t exit_code = cublasSetStream_v2(handle, stream); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasSetStream_v2 success, handle=%p, stream=%p\n", handle, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasSetStream_v2 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasSetWorkspace_v2Handle) { + tool::Logging(myName, "CUBLAS_SET_WORKSPACE_V2\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop()); + void* workspace = serverEp->GetDevPtr(reqBuf.Pop()); + size_t size = reqBuf.Pop(); + cublasStatus_t exit_code = cublasSetWorkspace_v2(handle, workspace, size); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasSetWorkspace_v2 success, handle=%p\n", handle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasSetWorkspace_v2 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasSetMathModeHandle) { + tool::Logging(myName, "CUBLAS_SET_MATH_MODE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cublasMath_t mode = reqBuf.Pop(); + cublasStatus_t exit_code = cublasSetMathMode(handle, mode); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasSetMathMode success, handle=%p\n", handle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasSetMathMode failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasGetMathModeHandle) { + tool::Logging(myName, "CUBLAS_GET_MATH_MODE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cublasMath_t mode; + cublasStatus_t exit_code = cublasGetMathMode(handle, &mode); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasGetMathMode success\n"); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLAS_GET_MATH_MODE); + resBuf.Push(mode); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasGetMathMode failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cuBlasLtCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cuBlasLtCallback.cc new file mode 100644 index 0000000..17d788e --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cuBlasLtCallback.cc @@ -0,0 +1,316 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "cuBlasLtHandle"; + +DEFINE_SERVER_AM_CALLBACK(cublasLtCreateHandle) { + tool::Logging(myName, "CUBLASLT_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cublasLtHandle_t lightHandle = NULL; + cublasStatus_t exit_code = cublasLtCreate(&lightHandle); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtCreate success, lightHandle = %p\n", lightHandle); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, lightHandle); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLASLT_CREATE); + lightHandle = (cublasLtHandle_t)serverEp->GetHandleVirAddr(lightHandle, CUBLASLT_CREATE); + resBuf.Push64BitPointer(lightHandle); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtCreate failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtDestroyHandle) { + tool::Logging(myName, "CUBLASLT_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtHandle_t lightHandle = (cublasLtHandle_t)serverEp->GetHandle(reqBuf.Pop(), true); + cublasStatus_t exit_code = cublasLtDestroy(lightHandle); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtDestroy success, lightHandle = %p\n", lightHandle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtDestroy failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulDescCreateHandle) { + tool::Logging(myName, "CUBLASLT_MATMULDESC_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cublasLtMatmulDesc_t matmulDesc; + cublasComputeType_t computeType = reqBuf.Pop(); + cudaDataType_t scaleType = reqBuf.Pop(); + cublasStatus_t exit_code = cublasLtMatmulDescCreate(&matmulDesc, computeType, scaleType); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulDescCreate success, matmulDesc = %p\n", matmulDesc); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, matmulDesc); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLASLT_MATMULDESC_CREATE); + matmulDesc = (cublasLtMatmulDesc_t)serverEp->GetHandleVirAddr(matmulDesc, CUBLASLT_MATMULDESC_CREATE); + resBuf.Push64BitPointer(matmulDesc); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulDescCreate failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulDescDestroyHandle) { + tool::Logging(myName, "CUBLASLT_MATMULDESC_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtMatmulDesc_t matmulDesc = (cublasLtMatmulDesc_t)serverEp->GetHandle(reqBuf.Pop(), true); + cublasStatus_t exit_code = cublasLtMatmulDescDestroy(matmulDesc); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulDescDestroy success, matmulDesc = %p\n", matmulDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulDescDestroy failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulDescSetAttributeHandle) { + tool::Logging(myName, "CUBLASLT_MATMULDESC_SETATTRIBUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtMatmulDesc_t matmulDesc = (cublasLtMatmulDesc_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatmulDescAttributes_t attr = reqBuf.Pop(); + const void* buf = reqBuf.AssignAddrForAll(); + size_t sizeInBytes = reqBuf.Pop(); + if (attr == CUBLASLT_MATMUL_DESC_BIAS_POINTER || attr == CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_POINTER + // || CUBLASLT_MATMUL_DESC_A_SCALE_POINTER || CUBLASLT_MATMUL_DESC_B_SCALE_POINTER || CUBLASLT_MATMUL_DESC_C_SCALE_POINTER || ... cuBlasLt v12.5 have more attributes related to devPtr + ) { + uint64_t* virtAddrPtr = (uint64_t*)buf; + void* devPtr = serverEp->GetDevPtr(*virtAddrPtr); + *virtAddrPtr = (uint64_t)devPtr; + } + + cublasStatus_t exit_code = cublasLtMatmulDescSetAttribute(matmulDesc, attr, buf, sizeInBytes); + + + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulDescSetAttribute success, desc = %p, attr = %d, buf = %p(val = %p), sizeInBytes = %lu\n", matmulDesc, attr, buf, *(uint64_t*)buf, sizeInBytes); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulDescSetAttribute failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatrixLayoutCreateHandle) { + tool::Logging(myName, "CUBLASLT_MATRIX_LAYOUT_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cublasLtMatrixLayout_t layout; + cudaDataType_t dataType = reqBuf.Pop(); + uint64_t rows = reqBuf.Pop(); + uint64_t cols = reqBuf.Pop(); + int64_t ld = reqBuf.Pop(); + cublasStatus_t exit_code = cublasLtMatrixLayoutCreate(&layout, dataType, rows, cols, ld); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatrixLayoutCreate success, layout = %p\n", layout); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, layout); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLASLT_MATRIX_LAYOUT_CREATE); + layout = (cublasLtMatrixLayout_t)serverEp->GetHandleVirAddr(layout, CUBLASLT_MATRIX_LAYOUT_CREATE); + resBuf.Push64BitPointer(layout); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatrixLayoutCreate failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatrixLayoutDestroyHandle) { + tool::Logging(myName, "CUBLASLT_MATRIX_LAYOUT_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtMatrixLayout_t layout = (cublasLtMatrixLayout_t)serverEp->GetHandle(reqBuf.Pop(), true); + cublasStatus_t exit_code = cublasLtMatrixLayoutDestroy(layout); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatrixLayoutDestroy success, layout = %p\n", layout); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatrixLayoutDestroy failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatrixLayoutSetAttributeHandle) { + tool::Logging(myName, "CUBLASLT_MATRIX_LAYOUT_SETATTRIBUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtMatrixLayout_t layout = (cublasLtMatrixLayout_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatrixLayoutAttribute_t attr = reqBuf.Pop(); + const void* buf = reqBuf.AssignAddrForAll(); + size_t sizeInBytes = reqBuf.Pop(); + cublasStatus_t exit_code = cublasLtMatrixLayoutSetAttribute(layout, attr, buf, sizeInBytes); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatrixLayoutSetAttribute success: layout = %p, attr = %d, buf = %p(val = %p), sizeInBytes = %lu\n", layout, attr, buf, *(uint64_t*)buf, sizeInBytes); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatrixLayoutSetAttribute failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulPreferenceCreateHandle) { + tool::Logging(myName, "CUBLASLT_MATMULPREFERENCE_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cublasLtMatmulPreference_t pref; + cublasStatus_t exit_code = cublasLtMatmulPreferenceCreate(&pref); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulPreferenceCreate success, pref = %p\n", pref); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, pref); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLASLT_MATMULPREFERENCE_CREATE); + pref = (cublasLtMatmulPreference_t)serverEp->GetHandleVirAddr(pref, CUBLASLT_MATMULPREFERENCE_CREATE); + resBuf.Push64BitPointer(pref); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulPreferenceCreate failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulPreferenceDestroyHandle) { + tool::Logging(myName, "CUBLASLT_MATMULPREFERENCE_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtMatmulPreference_t pref = (cublasLtMatmulPreference_t)serverEp->GetHandle(reqBuf.Pop(), true); + cublasStatus_t exit_code = cublasLtMatmulPreferenceDestroy(pref); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulPreferenceDestroy success, pref = %p\n", pref); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulPreferenceDestroy failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulPreferenceSetAttributeHandle) { + tool::Logging(myName, "CUBLASLT_MATMULPREFERENCE_SETATTRIBUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtMatmulPreference_t pref = (cublasLtMatmulPreference_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatmulPreferenceAttributes_t attr = reqBuf.Pop(); + const void* buf = reqBuf.AssignAddrForAll(); + size_t sizeInBytes = reqBuf.Pop(); + cublasStatus_t exit_code = cublasLtMatmulPreferenceSetAttribute(pref, attr, buf, sizeInBytes); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulPreferenceSetAttribute success: pref = %p, attr = %d, buf = %p(val = %p), sizeInBytes = %lu\n", pref, attr, buf, *(uint64_t*)buf, sizeInBytes); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulPreferenceSetAttribute failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulAlgoGetHeuristicHandle) { + tool::Logging(myName, "CUBLASLT_MATMULALGO_GETHEURISTIC\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtHandle_t lightHandle = (cublasLtHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatmulDesc_t matmulDesc = (cublasLtMatmulDesc_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatrixLayout_t layoutA = (cublasLtMatrixLayout_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatrixLayout_t layoutB = (cublasLtMatrixLayout_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatrixLayout_t layoutC = (cublasLtMatrixLayout_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatrixLayout_t layoutD = (cublasLtMatrixLayout_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatmulPreference_t pref = (cublasLtMatmulPreference_t)serverEp->GetHandle(reqBuf.Pop()); + int reqAlgoCount = reqBuf.Pop(); + cublasLtMatmulHeuristicResult_t heuristicResult[reqAlgoCount]; + int retAlgoCount = 0; + cublasStatus_t exit_code = cublasLtMatmulAlgoGetHeuristic(lightHandle, matmulDesc, layoutA, layoutB, layoutC, layoutD, pref, reqAlgoCount, heuristicResult, &retAlgoCount); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulAlgoGetHeuristic success: retAlgoCount = %d, lightHandle = %p, matmulDesc = %p, layoutA = %p, layoutB = %p, layoutC = %p, layoutD = %p, pref = %p\n", retAlgoCount, lightHandle, matmulDesc, layoutA, layoutB, layoutC, layoutD, pref); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLASLT_MATMULALGO_GETHEURISTIC); + resBuf.Push(retAlgoCount); + resBuf.Push(heuristicResult, retAlgoCount); + // heuristicResult contains cublasLtMatmulAlgo_t, which is a structure that describes the algorithm, and can be trivially serialized and later restored for use with the same version of cuBLAS library to save on selecting the right configuration again. + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulAlgoGetHeuristic failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulHandle) { + tool::Logging(myName, "CUBLASLT_MATMUL\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtHandle_t lightHandle = (cublasLtHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatmulDesc_t compDesc = (cublasLtMatmulDesc_t) serverEp->GetHandle(reqBuf.Pop()); + const float* alpha = reqBuf.AssignAddr(); // default: host pointer, set by CUBLASLT_MATMUL_DESC_POINTER_MODE + const void* A = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cublasLtMatrixLayout_t Adesc = (cublasLtMatrixLayout_t) serverEp->GetHandle(reqBuf.Pop()); + const void* B = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cublasLtMatrixLayout_t Bdesc = (cublasLtMatrixLayout_t) serverEp->GetHandle(reqBuf.Pop()); + const float* beta = reqBuf.AssignAddr(); + const void* C = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cublasLtMatrixLayout_t Cdesc = (cublasLtMatrixLayout_t) serverEp->GetHandle(reqBuf.Pop()); + void* D = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cublasLtMatrixLayout_t Ddesc = (cublasLtMatrixLayout_t) serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatmulAlgo_t* algo = reqBuf.AssignAddr(); + void* workspace = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + size_t workspaceSizeInBytes = reqBuf.Pop(); + cudaStream_t stream = (cudaStream_t) serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + cublasStatus_t exit_code = cublasLtMatmul(lightHandle, compDesc, alpha, A, Adesc, B, Bdesc, beta, C, Cdesc, D, Ddesc, algo, workspace, workspaceSizeInBytes, stream); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmul success\n"); + // cudaError_t exit_code_o = cudaDeviceSynchronize(); + // if (exit_code_o != cudaSuccess) { + // tool::Logging(LOG_ERROR, myName_, "cublasLtMatmul after cudaDeviceSynchronize failed: %d(%s)\n", exit_code_o, cudaGetErrorString(exit_code_o)); + // exit(EXIT_FAILURE); + // } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmul failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cuDNNCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cuDNNCallback.cc new file mode 100644 index 0000000..7f6b7a4 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cuDNNCallback.cc @@ -0,0 +1,1179 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "cuDNNHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudnnCreateHandle) { + tool::Logging(myName, "CUDNN_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudnnHandle_t handle = NULL; + cudnnStatus_t exit_code = cudnnCreate(&handle); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnCreate success, handle = %p\n", handle); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, handle); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_CREATE); + handle = (cudnnHandle_t)serverEp->GetHandleVirAddr(handle, CUDNN_CREATE); + resBuf.Push64BitPointer(handle); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnCreate failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnDestroyHandle) { + tool::Logging(myName, "CUDNN_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudnnStatus_t exit_code = cudnnDestroy(handle); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnDestroy success, handle = %p\n", handle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnDestroy failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnCreateTensorDescriptorHandle) { + tool::Logging(myName, "CUDNN_CREATE_TENSOR_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudnnTensorDescriptor_t tensorDesc = NULL; + cudnnStatus_t exit_code = cudnnCreateTensorDescriptor(&tensorDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnCreateTensorDescriptor success, tensorDesc = %p\n", tensorDesc); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, tensorDesc); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_CREATE_TENSOR_DESCRIPTOR); + tensorDesc = (cudnnTensorDescriptor_t)serverEp->GetHandleVirAddr(tensorDesc, CUDNN_CREATE_TENSOR_DESCRIPTOR); + resBuf.Push64BitPointer(tensorDesc); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnCreateTensorDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnDestroyTensorDescriptorHandle) { + tool::Logging(myName, "CUDNN_DESTROY_TENSOR_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorDescriptor_t tensorDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudnnStatus_t exit_code = cudnnDestroyTensorDescriptor(tensorDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnDestroyTensorDescriptor success, tensorDesc = %p\n", tensorDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnDestroyTensorDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetTensorSizeInBytesHandle) { + tool::Logging(myName, "CUDNN_GET_TENSOR_SIZE_IN_BYTES\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorDescriptor_t tensorDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetTensorSizeInBytes(tensorDesc, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetTensorSizeInBytes success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_TENSOR_SIZE_IN_BYTES); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetTensorSizeInBytes failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetTensor4dDescriptorHandle) { + tool::Logging(myName, "CUDNN_SET_TENSOR_4D_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorDescriptor_t tensorDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorFormat_t format = reqBuf.Pop(); + cudnnDataType_t dataType = reqBuf.Pop(); + int n = reqBuf.Pop(); + int c = reqBuf.Pop(); + int h = reqBuf.Pop(); + int w = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnSetTensor4dDescriptor(tensorDesc, format, dataType, n, c, h, w); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetTensor4dDescriptor success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetTensor4dDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetTensorNdDescriptorHandle) { + tool::Logging(myName, "CUDNN_SET_TENSOR_ND_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorDescriptor_t tensorDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnDataType_t dataType = reqBuf.Pop(); + int nbDims = reqBuf.Pop(); + const int* dimA = reqBuf.AssignAddr(); + const int* strideA = reqBuf.AssignAddr(); + cudnnStatus_t exit_code = cudnnSetTensorNdDescriptor(tensorDesc, dataType, nbDims, dimA, strideA); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetTensorNdDescriptor success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetTensorNdDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetTensorNdDescriptorExHandle) { + tool::Logging(myName, "CUDNN_SET_TENSOR_ND_DESCRIPTOR_EX\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorDescriptor_t tensorDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorFormat_t format = reqBuf.Pop(); + cudnnDataType_t dataType = reqBuf.Pop(); + int nbDims = reqBuf.Pop(); + const int* dimA = reqBuf.AssignAddr(); + cudnnStatus_t exit_code = cudnnSetTensorNdDescriptorEx(tensorDesc, format, dataType, nbDims, dimA); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetTensorNdDescriptorEx success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetTensorNdDescriptorEx failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnCreateTensorTransformDescriptorHandle) { + tool::Logging(myName, "CUDNN_CREATE_TENSOR_TRANSFORM_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudnnTensorTransformDescriptor_t transformDesc = NULL; + cudnnStatus_t exit_code = cudnnCreateTensorTransformDescriptor(&transformDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnCreateTensorTransformDescriptor success, transformDesc = %p\n", transformDesc); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, transformDesc); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_CREATE_TENSOR_TRANSFORM_DESCRIPTOR); + transformDesc = (cudnnTensorTransformDescriptor_t)serverEp->GetHandleVirAddr(transformDesc, CUDNN_CREATE_TENSOR_TRANSFORM_DESCRIPTOR); + resBuf.Push64BitPointer(transformDesc); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnCreateTensorTransformDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetTensorTransformDescriptorHandle) { + tool::Logging(myName, "CUDNN_SET_TENSOR_TRANSFORM_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorTransformDescriptor_t transformDesc = (cudnnTensorTransformDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const uint32_t nbDims = reqBuf.Pop(); + const cudnnTensorFormat_t destFormat = reqBuf.Pop(); + const int32_t* padBeforeA = reqBuf.AssignAddr(); + const int32_t* padAfterA = reqBuf.AssignAddr(); + const uint32_t* foldA = reqBuf.AssignAddr(); // spatial dimension (dimensions 2 and up) + const cudnnFoldingDirection_t dirt = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnSetTensorTransformDescriptor(transformDesc, nbDims, destFormat, padBeforeA, padAfterA, foldA, dirt); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetTensorTransformDescriptor success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetTensorTransformDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnDestroyTensorTransformDescriptorHandle) { + tool::Logging(myName, "CUDNN_DESTROY_TENSOR_TRANSFORM_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorTransformDescriptor_t transformDesc = (cudnnTensorTransformDescriptor_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudnnStatus_t exit_code = cudnnDestroyTensorTransformDescriptor(transformDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnDestroyTensorTransformDescriptor success, transformDesc = %p\n", transformDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnDestroyTensorTransformDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnInitTransformDestHandle) { + tool::Logging(myName, "CUDNN_INIT_TRANSFORM_DEST\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorTransformDescriptor_t transformDesc = (cudnnTensorTransformDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t srcDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t destDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + size_t destSizeInBytes = 0; + cudnnStatus_t exit_code = cudnnInitTransformDest(transformDesc, srcDesc, destDesc, &destSizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnInitTransformDest success, destSizeInBytes = %lu, transformDesc = %p, srcDesc = %p, destDesc = %p\n", destSizeInBytes, transformDesc, srcDesc, destDesc); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_INIT_TRANSFORM_DEST); + resBuf.Push(destSizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnInitTransformDest failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } + +} + +DEFINE_SERVER_AM_CALLBACK(cudnnTransformTensorExHandle) { + tool::Logging(myName, "CUDNN_TRANSFORM_TENSOR_EX\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorTransformDescriptor_t transDesc + = (const cudnnTensorTransformDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const void* alpha = reqBuf.AssignAddr(); + const cudnnTensorDescriptor_t srDesc= (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* srcData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* beta = reqBuf.AssignAddr(); + const cudnnTensorDescriptor_t dsDesc= (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* destData = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnTransformTensorEx(handle, transDesc, alpha, srDesc, srcData, beta, dsDesc, destData); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnTransformTensorEx success, handle = %p, transDesc = %p, srDesc = %p, dsDesc = %p\n", handle, transDesc, srDesc, dsDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnTransformTensorEx failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnTransformFilterHandle) { + tool::Logging(myName, "CUDNN_TRANSFORM_FILTER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorTransformDescriptor_t transDesc + = (const cudnnTensorTransformDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const void* alpha = reqBuf.AssignAddr(); + const cudnnFilterDescriptor_t srDesc= (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* srcData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* beta = reqBuf.AssignAddr(); + const cudnnFilterDescriptor_t dsDesc= (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* destData = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnTransformFilter(handle, transDesc, alpha, srDesc, srcData, beta, dsDesc, destData); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnTransformFilter success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnTransformFilter failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnCreateFilterDescriptorHandle) { + tool::Logging(myName, "CUDNN_CREATE_FILTER_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudnnFilterDescriptor_t filterDesc = NULL; + cudnnStatus_t exit_code = cudnnCreateFilterDescriptor(&filterDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnCreateFilterDescriptor success, filterDesc = %p\n", filterDesc); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, filterDesc); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_CREATE_FILTER_DESCRIPTOR); + filterDesc = (cudnnFilterDescriptor_t)serverEp->GetHandleVirAddr(filterDesc, CUDNN_CREATE_FILTER_DESCRIPTOR); + resBuf.Push64BitPointer(filterDesc); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnCreateFilterDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetFilterNdDescriptorHandle) { + tool::Logging(myName, "CUDNN_SET_FILTER_ND_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnFilterDescriptor_t filterDesc = (cudnnFilterDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnDataType_t dataType = reqBuf.Pop(); + cudnnTensorFormat_t format = reqBuf.Pop(); + int nbDims = reqBuf.Pop(); + const int* filterDimA = reqBuf.AssignAddr(); + cudnnStatus_t exit_code = cudnnSetFilterNdDescriptor(filterDesc, dataType, format, nbDims, filterDimA); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetFilterNdDescriptor success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetFilterNdDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnDestroyFilterDescriptorHandle) { + tool::Logging(myName, "CUDNN_DESTROY_FILTER_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnFilterDescriptor_t filterDesc = (cudnnFilterDescriptor_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudnnStatus_t exit_code = cudnnDestroyFilterDescriptor(filterDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnDestroyFilterDescriptor success, filterDesc = %p\n", filterDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnDestroyFilterDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetFilterSizeInBytesHandle) { + tool::Logging(myName, "CUDNN_GET_FILTER_SIZE_IN_BYTES\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnFilterDescriptor_t filterDesc = (cudnnFilterDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + size_t sizeInBytes= 0; + cudnnStatus_t exit_code = cudnnGetFilterSizeInBytes(filterDesc, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetFilterSizeInBytes success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_FILTER_SIZE_IN_BYTES); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetFilterSizeInBytes failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetFoldedConvBackwardDataDescriptorsHandle) { + tool::Logging(myName, "CUDNN_GET_FOLDED_CONV_BACKWARD_DATA_DESCRIPTORS\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnFilterDescriptor_t filterDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t diffDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t gradDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorFormat_t transformFormat = reqBuf.Pop(); + cudnnFilterDescriptor_t foldedFilterDesc = (cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t paddedDiffDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionDescriptor_t foldedConvDesc = (cudnnConvolutionDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t foldedGradDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorTransformDescriptor_t filterFoldTransDesc = (cudnnTensorTransformDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorTransformDescriptor_t diffPadTransDesc = (cudnnTensorTransformDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorTransformDescriptor_t gradFoldTransDesc = (cudnnTensorTransformDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorTransformDescriptor_t gradUnfoldTransDesc = (cudnnTensorTransformDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnGetFoldedConvBackwardDataDescriptors(handle, filterDesc, diffDesc, convDesc, gradDesc, transformFormat, foldedFilterDesc, paddedDiffDesc, foldedConvDesc, foldedGradDesc, filterFoldTransDesc, diffPadTransDesc, gradFoldTransDesc, gradUnfoldTransDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetFoldedConvBackwardDataDescriptors success, handle = %p, filterDesc = %p, diffDesc = %p, convDesc = %p, gradDesc = %p, transformFormat = %d, foldedFilterDesc = %p, paddedDiffDesc = %p, foldedConvDesc = %p, foldedGradDesc = %p, filterFoldTransDesc = %p, diffPadTransDesc = %p, gradFoldTransDesc = %p, gradUnfoldTransDesc = %p\n", handle, filterDesc, diffDesc, convDesc, gradDesc, transformFormat, foldedFilterDesc, paddedDiffDesc, foldedConvDesc, foldedGradDesc, filterFoldTransDesc, diffPadTransDesc, gradFoldTransDesc, gradUnfoldTransDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetFoldedConvBackwardDataDescriptors failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetStreamHandle) { + tool::Logging(myName, "CUDNN_SET_STREAM\n"); + ServerEndpoint* serverEp= (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t) serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + cudnnStatus_t exit_code = cudnnSetStream(handle, stream); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetStream success, handle = %p, stream = %p\n", handle, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetStream failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBatchNormalizationBackwardExHandle) { + tool::Logging(myName, "CUDNN_BATCH_NORMALIZATION_BACKWARD_EX\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBatchNormMode_t mode = reqBuf.Pop(); + cudnnBatchNormOps_t bnOps = reqBuf.Pop(); + const void* alphaDataDiff = reqBuf.AssignAddr(); // always one element + const void* betaDataDiff = reqBuf.AssignAddr(); // always one element + const void* alphaParamDiff= reqBuf.AssignAddr(); // always one element + const void* betaParamDiff = reqBuf.AssignAddr(); // always one element + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* xData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t yDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* yData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t dyDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* dyData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t dzDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* dzData = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t dxDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* dxData = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t dBnScaleBiasDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const void* bnScaleData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* bnBiasData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + void* dBnScaleData= (void*) serverEp->GetDevPtr(reqBuf.Pop()); + void* dBnBiasData = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + double epsilon = reqBuf.Pop(); + const void* savedMean = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* savedInvVariance = (const void*)serverEp->GetDevPtr(reqBuf.Pop()); + cudnnActivationDescriptor_t activationDesc = (cudnnActivationDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + void* workSpace = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + size_t workSpaceSizeInBytes = reqBuf.Pop(); + void* reserveSpace= (void*) serverEp->GetDevPtr(reqBuf.Pop()); + size_t reserveSpaceSizeInBytes = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnBatchNormalizationBackwardEx(handle, mode, bnOps, + alphaDataDiff, betaDataDiff, alphaParamDiff, betaParamDiff, + xDesc, xData, yDesc, yData, dyDesc, dyData, dzDesc, dzData, dxDesc, dxData, + dBnScaleBiasDesc, bnScaleData, bnBiasData, dBnScaleData, dBnBiasData, + epsilon, savedMean, savedInvVariance, activationDesc, + workSpace, workSpaceSizeInBytes, reserveSpace, reserveSpaceSizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBatchNormalizationBackwardEx success, handle = %p, xDesc = %p, yDesc = %p, dyDesc = %p, dzDesc = %p, dxDesc = %p, dBnScaleBiasDesc = %p\n", handle, xDesc, yDesc, dyDesc, dzDesc, dxDesc, dBnScaleBiasDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBatchNormalizationBackwardEx failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBatchNormalizationForwardTrainingExHandle) { + tool::Logging(myName, "CUDNN_BATCH_NORMALIZATION_FORWARD_TRAINING_EX\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBatchNormMode_t mode = reqBuf.Pop(); + cudnnBatchNormOps_t bnOps = reqBuf.Pop(); + const float* alphaData = reqBuf.AssignAddr(); // always one element + const float* betaData = reqBuf.AssignAddr(); // always one element + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* xData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t zDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* zData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t yDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* yData = serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t bnScaleBiasMeanVarDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* bnScaleData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* bnBiasData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + double exponentialAverageFactor = reqBuf.Pop(); + void* resultRunningMean = serverEp->GetDevPtr(reqBuf.Pop()); + void* resultRunningVariance = serverEp->GetDevPtr(reqBuf.Pop()); + double epsilon = reqBuf.Pop(); + void* saveMean = serverEp->GetDevPtr(reqBuf.Pop()); + void* saveInvVari = serverEp->GetDevPtr(reqBuf.Pop()); + cudnnActivationDescriptor_t activationDesc = (cudnnActivationDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* workSpace = serverEp->GetDevPtr(reqBuf.Pop()); + size_t workSpaceSizeInBytes = reqBuf.Pop(); + void* reserveSpace= serverEp->GetDevPtr(reqBuf.Pop()); + size_t reserveSpaceSizeInBytes = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnBatchNormalizationForwardTrainingEx(handle, mode, bnOps, alphaData, betaData, xDesc, xData, zDesc, zData, yDesc, yData, bnScaleBiasMeanVarDesc, bnScaleData, bnBiasData, exponentialAverageFactor, resultRunningMean, resultRunningVariance, epsilon, saveMean, saveInvVari, activationDesc, workSpace, workSpaceSizeInBytes, reserveSpace, reserveSpaceSizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBatchNormalizationForwardTrainingEx success, handle = %p, xDesc = %p, zDesc = %p, yDesc = %p, bnScaleBiasMeanVarDesc = %p\n", handle, xDesc, zDesc, yDesc, bnScaleBiasMeanVarDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBatchNormalizationForwardTrainingEx failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBatchNormalizationForwardInferenceHandle) { + tool::Logging(myName, "CUDNN_BATCH_NORMALIZATION_FORWARD_INFERENCE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBatchNormMode_t mode = reqBuf.Pop(); + const float* alpha = reqBuf.AssignAddr(); // always one element + const float* beta = reqBuf.AssignAddr(); // always one element + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* x = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t yDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* y = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t bnScaleBiasMeanVarDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const void* bnScale = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* bnBias = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* estimatedMean = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* estimatedVariance = (const void*)serverEp->GetDevPtr(reqBuf.Pop()); + double epsilon = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnBatchNormalizationForwardInference(handle, mode, alpha, beta, xDesc, x, yDesc, y, bnScaleBiasMeanVarDesc, bnScale, bnBias, estimatedMean, estimatedVariance, epsilon); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBatchNormalizationForwardInference success, handle = %p, xDesc = %p, yDesc = %p, bnScaleBiasMeanVarDesc = %p\n", handle, xDesc, yDesc, bnScaleBiasMeanVarDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBatchNormalizationForwardInference failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBackendCreateDescriptorHandle) { + tool::Logging(myName, "CUDNN_BACKEND_CREATE_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnBackendDescriptorType_t decType = reqBuf.Pop(); + uint64_t virAddr = reqBuf.Pop(); + cudnnBackendDescriptor_t backendDesc = NULL; + cudnnStatus_t exit_code = cudnnBackendCreateDescriptor(decType, &backendDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBackendCreateDescriptor success, backendDesc = %p\n", backendDesc); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, backendDesc); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_BACKEND_CREATE_DESCRIPTOR); + backendDesc = (cudnnBackendDescriptor_t)serverEp->GetHandleVirAddr(backendDesc, CUDNN_BACKEND_CREATE_DESCRIPTOR); + resBuf.Push64BitPointer(backendDesc); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBackendCreateDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBackendDestroyDescriptorHandle) { + tool::Logging(myName, "CUDNN_BACKEND_DESTROY_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnBackendDescriptor_t backendDesc = (cudnnBackendDescriptor_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudnnStatus_t exit_code = cudnnBackendDestroyDescriptor(backendDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBackendDestroyDescriptor success, backendDesc = %p\n", backendDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBackendDestroyDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBackendSetAttributeHandle) { // todo: to be validated + tool::Logging(myName, "CUDNN_BACKEND_SET_ATTRIBUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnBackendDescriptor_t backendDesc = (cudnnBackendDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnBackendAttributeName_t attrName = reqBuf.Pop(); + cudnnBackendAttributeType_t attrType = reqBuf.Pop(); + int64_t eleCount = reqBuf.Pop(); + const uint8_t* arrayOfElem = reqBuf.AssignAddrForAll(); // size is eleCount * sizeof(...) + + if (attrType == CUDNN_TYPE_BACKEND_DESCRIPTOR || attrType == CUDNN_TYPE_HANDLE + || attrName == CUDNN_ATTR_EXECUTION_PLAN_COMPUTED_INTERMEDIATE_UIDS + || attrName == CUDNN_ATTR_EXECUTION_PLAN_RUN_ONLY_INTERMEDIATE_UIDS + || attrName == CUDNN_ATTR_INTERMEDIATE_INFO_UNIQUE_ID + || attrName == CUDNN_ATTR_INTERMEDIATE_INFO_DEPENDENT_DATA_UIDS + || attrName == CUDNN_ATTR_TENSOR_UNIQUE_ID + || attrName == CUDNN_ATTR_VARIANT_PACK_UNIQUE_IDS + || attrName == CUDNN_ATTR_LAYOUT_INFO_TENSOR_UID + ) { + uint64_t* ptrList = (uint64_t*)arrayOfElem; + for (int i = 0; i < eleCount; i++) { + ptrList[i] = (uint64_t)serverEp->GetHandle(ptrList[i]); + } + } + + if (attrType == CUDNN_TYPE_VOID_PTR) { + uint64_t* ptrList = (uint64_t*)arrayOfElem; + for (int i = 0; i < eleCount; i++) { + ptrList[i] = (uint64_t)serverEp->GetDevPtr(ptrList[i]); + } + } + + cudnnStatus_t exit_code = cudnnBackendSetAttribute(backendDesc, attrName, attrType, eleCount, arrayOfElem); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBackendSetAttribute success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBackendSetAttribute failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBackendGetAttributeHandle) { // todo: to be validated + tool::Logging(myName, "CUDNN_BACKEND_GET_ATTRIBUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnBackendDescriptor_t backendDesc = (cudnnBackendDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnBackendAttributeName_t attrName = reqBuf.Pop(); + cudnnBackendAttributeType_t attrType = reqBuf.Pop(); + int64_t reqElemCount = reqBuf.Pop(); + size_t varLen = reqBuf.Pop(); + // uint8_t* arrayOfElem = (uint8_t*)malloc(varLen * reqElemCount); + uint8_t arrayOfElem[varLen * reqElemCount]; + int64_t elementCount = 0; + cudnnStatus_t exit_code = cudnnBackendGetAttribute(backendDesc, attrName, attrType, reqElemCount, &elementCount, arrayOfElem); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBackendSetAttribute success\n"); + + if (attrType == CUDNN_TYPE_BACKEND_DESCRIPTOR || attrType == CUDNN_TYPE_HANDLE + || attrName == CUDNN_ATTR_EXECUTION_PLAN_COMPUTED_INTERMEDIATE_UIDS + || attrName == CUDNN_ATTR_EXECUTION_PLAN_RUN_ONLY_INTERMEDIATE_UIDS + || attrName == CUDNN_ATTR_INTERMEDIATE_INFO_UNIQUE_ID + || attrName == CUDNN_ATTR_INTERMEDIATE_INFO_DEPENDENT_DATA_UIDS + || attrName == CUDNN_ATTR_TENSOR_UNIQUE_ID + || attrName == CUDNN_ATTR_VARIANT_PACK_UNIQUE_IDS + || attrName == CUDNN_ATTR_LAYOUT_INFO_TENSOR_UID + ) { + uint64_t* ptrList = (uint64_t*)arrayOfElem; + for (int i = 0; i < elementCount; i++) { + ptrList[i] = serverEp->FindHandleVirAddr((void*)ptrList[i]); + } + } + + if (attrType == CUDNN_TYPE_VOID_PTR) { + uint64_t* ptrList = (uint64_t*)arrayOfElem; + for (int i = 0; i < elementCount; i++) { + ptrList[i] = serverEp->FindDevPtrVirAddr((void*)ptrList[i]); + } + } + + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_BACKEND_GET_ATTRIBUTE); + resBuf.Push(elementCount); + resBuf.Push(arrayOfElem, varLen * reqElemCount); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBackendSetAttribute failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBackendExecuteHandle) { + tool::Logging(myName, "CUDNN_BACKEND_EXECUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBackendDescriptor_t execPlan = (cudnnBackendDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnBackendDescriptor_t varianPack = (cudnnBackendDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnBackendExecute(handle, execPlan, varianPack); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBackendExecute success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBackendExecute failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBackendFinalizeHandle) { + tool::Logging(myName, "CUDNN_BACKEND_FINALIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnBackendDescriptor_t descriptor = (cudnnBackendDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnBackendFinalize(descriptor); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBackendFinalize success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBackendFinalize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetBatchNormalizationBackwardExWorkspaceSizeHandle) { + tool::Logging(myName, "CUDNN_GET_BATCH_NORMALIZATION_BACKWARD_EX_WORKSPACE_SIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBatchNormMode_t mode = reqBuf.Pop(); + cudnnBatchNormOps_t bnOps = reqBuf.Pop(); + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t yDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dyDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dzDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dxDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dBnScaleBiasDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnActivationDescriptor_t activationDesc = (cudnnActivationDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetBatchNormalizationBackwardExWorkspaceSize(handle, mode, bnOps, xDesc, yDesc, dyDesc, dzDesc, dxDesc, dBnScaleBiasDesc, activationDesc, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetBatchNormalizationBackwardExWorkspaceSize success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_BATCH_NORMALIZATION_BACKWARD_EX_WORKSPACE_SIZE); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetBatchNormalizationBackwardExWorkspaceSize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetBatchNormalizationForwardTrainingExWorkspaceSizeHandle) { + tool::Logging(myName, "CUDNN_GET_BATCH_NORMALIZATION_FORWARD_TRAINING_EX_WORKSPACE_SIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBatchNormMode_t mode = reqBuf.Pop(); + cudnnBatchNormOps_t bnOps = reqBuf.Pop(); + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t zDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t yDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t bnScaleBiasMeanVarDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnActivationDescriptor_t activationDesc = (cudnnActivationDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize(handle, mode, bnOps, xDesc, zDesc, yDesc, bnScaleBiasMeanVarDesc, activationDesc, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_BATCH_NORMALIZATION_FORWARD_TRAINING_EX_WORKSPACE_SIZE); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetBatchNormalizationTrainingExReserveSpaceSizeHandle) { + tool::Logging(myName, "CUDNN_GET_BATCH_NORMALIZATION_TRAINING_EX_RESERVE_SPACE_SIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBatchNormMode_t mode = reqBuf.Pop(); + cudnnBatchNormOps_t bnOps = reqBuf.Pop(); + cudnnActivationDescriptor_t activationDesc = (cudnnActivationDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetBatchNormalizationTrainingExReserveSpaceSize(handle, mode, bnOps, activationDesc, xDesc, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetBatchNormalizationTrainingExReserveSpaceSize success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_BATCH_NORMALIZATION_TRAINING_EX_RESERVE_SPACE_SIZE); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetBatchNormalizationTrainingExReserveSpaceSize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnCreateConvolutionDescriptorHandle) { + tool::Logging(myName, "CUDNN_CREATE_CONVOLUTION_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudnnConvolutionDescriptor_t convDesc = NULL; + cudnnStatus_t exit_code = cudnnCreateConvolutionDescriptor(&convDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnCreateConvolutionDescriptor success, convDesc = %p\n", convDesc); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, convDesc); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_CREATE_CONVOLUTION_DESCRIPTOR); + convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandleVirAddr(convDesc, CUDNN_CREATE_CONVOLUTION_DESCRIPTOR); + resBuf.Push64BitPointer(convDesc); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnCreateConvolutionDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnDestroyConvolutionDescriptorHandle) { + tool::Logging(myName, "CUDNN_DESTROY_CONVOLUTION_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudnnStatus_t exit_code = cudnnDestroyConvolutionDescriptor(convDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnDestroyConvolutionDescriptor success, convDesc = %p\n", convDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnDestroyConvolutionDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetConvolutionGroupCountHandle) { + tool::Logging(myName, "CUDNN_SET_CONVOLUTION_GROUP_COUNT\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + int groupCount = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnSetConvolutionGroupCount(convDesc, groupCount); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetConvolutionGroupCount success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetConvolutionGroupCount failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetConvolutionMathTypeHandle) { + tool::Logging(myName, "CUDNN_SET_CONVOLUTION_MATH_TYPE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnMathType_t mathType = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnSetConvolutionMathType(convDesc, mathType); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetConvolutionMathType success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetConvolutionMathType failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetConvolutionNdDescriptorHandle) { + tool::Logging(myName, "CUDNN_SET_CONVOLUTION_ND_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + int arrayLength= reqBuf.Pop(); + const int* padA = reqBuf.AssignAddr(); + const int* filterStrideA = reqBuf.AssignAddr(); + const int* dilationA = reqBuf.AssignAddr(); + cudnnConvolutionMode_t mode = reqBuf.Pop(); + cudnnDataType_t dataType = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnSetConvolutionNdDescriptor(convDesc, arrayLength, padA, filterStrideA, dilationA, mode, dataType); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetConvolutionNdDescriptor success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetConvolutionNdDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetConvolutionReorderTypeHandle) { + tool::Logging(myName, "CUDNN_SET_CONVOLUTION_REORDER_TYPE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnReorderType_t reorderType = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnSetConvolutionReorderType(convDesc, reorderType); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetConvolutionReorderType success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetConvolutionReorderType failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetConvolutionForwardAlgorithm_v7Handle) { + tool::Logging(myName, "CUDNN_GET_CONVOLUTION_FORWARD_ALGORITHM_V7\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnFilterDescriptor_t wDesc = (cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t yDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const int requestAlgoCount = reqBuf.Pop(); + int returnedAlgoCount = 0; + // cudnnConvolutionFwdAlgoPerf_t *perfResults = (cudnnConvolutionFwdAlgoPerf_t*)malloc(requestAlgoCount * sizeof(cudnnConvolutionFwdAlgoPerf_t)); + cudnnConvolutionFwdAlgoPerf_t perfResults[requestAlgoCount]; + cudnnStatus_t exit_code = cudnnGetConvolutionForwardAlgorithm_v7(handle, xDesc, wDesc, convDesc, yDesc, requestAlgoCount, &returnedAlgoCount, perfResults); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetConvolutionForwardAlgorithm_v7 success, returnedAlgoCount = %d\n", returnedAlgoCount); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_CONVOLUTION_FORWARD_ALGORITHM_V7); + resBuf.Push(returnedAlgoCount); + resBuf.Push(perfResults, requestAlgoCount); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetConvolutionForwardAlgorithm_v7 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetConvolutionBackwardFilterAlgorithm_v7Handle) { + tool::Logging(myName, "CUDNN_GET_CONVOLUTION_BACKWARD_FILTER_ALGORITHM_V7\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dyDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnFilterDescriptor_t dwDesc = (cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const int requestAlgoCount = reqBuf.Pop(); + int returnedAlgoCount = 0; + // cudnnConvolutionBwdFilterAlgoPerf_t *perfResults = (cudnnConvolutionBwdFilterAlgoPerf_t*)malloc(requestAlgoCount * sizeof(cudnnConvolutionBwdFilterAlgoPerf_t)); + cudnnConvolutionBwdFilterAlgoPerf_t perfResults[requestAlgoCount]; + cudnnStatus_t exit_code = cudnnGetConvolutionBackwardFilterAlgorithm_v7(handle, xDesc, dyDesc, convDesc, dwDesc, requestAlgoCount, &returnedAlgoCount, perfResults); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetConvolutionBackwardFilterAlgorithm_v7 success, returnedAlgoCount = %d\n", returnedAlgoCount); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_CONVOLUTION_BACKWARD_FILTER_ALGORITHM_V7); + resBuf.Push(returnedAlgoCount); + resBuf.Push(perfResults, requestAlgoCount); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetConvolutionBackwardFilterAlgorithm_v7 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetConvolutionBackwardDataAlgorithm_v7Handle) { + tool::Logging(myName, "CUDNN_GET_CONVOLUTION_BACKWARD_DATA_ALGORITHM_V7\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnFilterDescriptor_t wDesc = (cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dyDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dxDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const int requestAlgoCount = reqBuf.Pop(); + int returnedAlgoCount = 0; + // cudnnConvolutionBwdDataAlgoPerf_t *perfResults = (cudnnConvolutionBwdDataAlgoPerf_t*)malloc(requestAlgoCount * sizeof(cudnnConvolutionBwdDataAlgoPerf_t)); + cudnnConvolutionBwdDataAlgoPerf_t perfResults[requestAlgoCount]; + cudnnStatus_t exit_code = cudnnGetConvolutionBackwardDataAlgorithm_v7(handle, wDesc, dyDesc, convDesc, dxDesc, requestAlgoCount, &returnedAlgoCount, perfResults); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetConvolutionBackwardDataAlgorithm_v7 success, returnedAlgoCount = %d\n", returnedAlgoCount); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_CONVOLUTION_BACKWARD_DATA_ALGORITHM_V7); + resBuf.Push(returnedAlgoCount); + resBuf.Push(perfResults, requestAlgoCount); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetConvolutionBackwardDataAlgorithm_v7 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetConvolutionForwardWorkspaceSizeHandle) { + tool::Logging(myName, "CUDNN_GET_CONVOLUTION_FORWARD_WORKSPACE_SIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t xDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnFilterDescriptor_t wDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t yDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionFwdAlgo_t algo = reqBuf.Pop(); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetConvolutionForwardWorkspaceSize(handle, xDesc, wDesc, convDesc, yDesc, algo, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetConvolutionForwardWorkspaceSize success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_CONVOLUTION_FORWARD_WORKSPACE_SIZE); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetConvolutionForwardWorkspaceSize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnConvolutionForwardHandle) { + tool::Logging(myName, "CUDNN_CONVOLUTION_FORWARD\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const void* alpha = reqBuf.AssignAddr(); // always one element + const cudnnTensorDescriptor_t xDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* x = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const cudnnFilterDescriptor_t wDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* w = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionFwdAlgo_t algo = reqBuf.Pop(); + void* workSpace = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + size_t workSpaceSizeInBytes = reqBuf.Pop(); + const void* beta = reqBuf.AssignAddr(); // always one element + const cudnnTensorDescriptor_t yDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* y = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnConvolutionForward(handle, alpha, xDesc, x, wDesc, w, convDesc, algo, workSpace, workSpaceSizeInBytes, beta, yDesc, y); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnConvolutionForward success, handle = %p, xDesc = %p, wDesc = %p, convDesc = %p, yDesc = %p\n", handle, xDesc, wDesc, convDesc, yDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnConvolutionForward failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetConvolutionBackwardDataWorkspaceSizeHandle) { + tool::Logging(myName, "CUDNN_GET_CONVOLUTION_BACKWARD_DATA_WORKSPACE_SIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnFilterDescriptor_t wDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t dyDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t dxDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionBwdDataAlgo_t algo = reqBuf.Pop(); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetConvolutionBackwardDataWorkspaceSize(handle, wDesc, dyDesc, convDesc, dxDesc, algo, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetConvolutionBackwardDataWorkspaceSize success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_CONVOLUTION_BACKWARD_DATA_WORKSPACE_SIZE); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetConvolutionBackwardDataWorkspaceSize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnConvolutionBackwardFilterHandle) { + tool::Logging(myName, "CUDNN_CONVOLUTION_BACKWARD_FILTER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const void* alpha = reqBuf.AssignAddr(); // always one element + const cudnnTensorDescriptor_t xDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* x = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const cudnnTensorDescriptor_t dyDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* dy = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionBwdFilterAlgo_t algo = reqBuf.Pop(); + void* workSpace = serverEp->GetDevPtr(reqBuf.Pop()); + size_t workSpaceSizeInBytes = reqBuf.Pop(); + const void* beta = reqBuf.AssignAddr(); // always one element + const cudnnFilterDescriptor_t dwDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* dw = serverEp->GetDevPtr(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnConvolutionBackwardFilter(handle, alpha, xDesc, x, dyDesc, dy, convDesc, algo, workSpace, workSpaceSizeInBytes, beta, dwDesc, dw); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnConvolutionBackwardFilter success, handle = %p, xDesc = %p, dyDesc = %p, convDesc = %p, dwDesc = %p\n", handle, xDesc, dyDesc, convDesc, dwDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnConvolutionBackwardFilter failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetConvolutionBackwardFilterWorkspaceSizeHandle) { + tool::Logging(myName, "CUDNN_GET_CONVOLUTION_BACKWARD_FILTER_WORKSPACE_SIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t xDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t dyDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const cudnnFilterDescriptor_t dwDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionBwdFilterAlgo_t algo = reqBuf.Pop(); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetConvolutionBackwardFilterWorkspaceSize(handle, xDesc, dyDesc, convDesc, dwDesc, algo, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetConvolutionBackwardFilterWorkspaceSize success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_CONVOLUTION_BACKWARD_FILTER_WORKSPACE_SIZE); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetConvolutionBackwardFilterWorkspaceSize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnConvolutionBackwardDataHandle) { + tool::Logging(myName, "CUDNN_CONVOLUTION_BACKWARD_DATA\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const void* alpha = reqBuf.AssignAddr(); // always one element + const cudnnFilterDescriptor_t wDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* w = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const cudnnTensorDescriptor_t dyDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* dy = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionBwdDataAlgo_t algo = reqBuf.Pop(); + void* workSpace = serverEp->GetDevPtr(reqBuf.Pop()); + size_t workSpaceSizeInBytes = reqBuf.Pop(); + const void* beta = reqBuf.AssignAddr(); // always one element + const cudnnTensorDescriptor_t dxDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* dx = serverEp->GetDevPtr(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnConvolutionBackwardData(handle, alpha, wDesc, w, dyDesc, dy, convDesc, algo, workSpace, workSpaceSizeInBytes, beta, dxDesc, dx); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnConvolutionBackwardData success, handle = %p, wDesc = %p, dyDesc = %p, convDesc = %p, dxDesc = %p\n", handle, wDesc, dyDesc, convDesc, dxDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnConvolutionBackwardData failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtDeviceCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtDeviceCallback.cc new file mode 100644 index 0000000..4be4fd3 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtDeviceCallback.cc @@ -0,0 +1,155 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "CUDARuntimeDeviceHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudaGetDeviceHandle) { + tool::Logging(myName, "CUDA_GET_DEVICE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int device = 0; + + cudaError_t exit_code = cudaGetDevice(&device); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaGetDevice success, device = %d\n", device); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_GET_DEVICE); + resBuf.Push(device); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaGetDevice failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaGetDeviceCountHandle) { + tool::Logging(myName, "CUDA_GET_DEVICE_COUNT\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int count = 0; + + cudaError_t exit_code = cudaGetDeviceCount(&count); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaGetDeviceCount success, count = %d\n", count); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_GET_DEVICE_COUNT); + resBuf.Push(count); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaGetDeviceCount failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaGetDevicePropertiesHandle) { + tool::Logging(myName, "CUDA_GET_DEVICE_PROPERTIES\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int device = reqBuf.Pop(); + cudaDeviceProp prop; + + cudaError_t exit_code = cudaGetDeviceProperties(&prop, device); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaGetDeviceProperties success, device = %d\n", device); + prop.canMapHostMemory = 0; //! disable UVA + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_GET_DEVICE_PROPERTIES); + resBuf.Push(prop); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaGetDeviceProperties failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaDeviceGetAttributeHandle) { + tool::Logging(myName, "CUDA_DEVICE_GET_ATTRIBUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int value = 0; + cudaDeviceAttr attr = (cudaDeviceAttr)reqBuf.Pop(); + int device = reqBuf.Pop(); + + cudaError_t exit_code = cudaDeviceGetAttribute(&value, attr, device); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaDeviceGetAttribute success, device = %d, attr = %d, value = %d\n", device, attr, value); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_DEVICE_GET_ATTRIBUTE); + resBuf.Push(value); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaDeviceGetAttribute failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaSetDeviceHandle) { + tool::Logging(myName, "CUDA_SET_DEVICE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int deviceIdx = reqBuf.Pop(); //! index of devList, not device id + + cudaError_t exit_code = cudaSetDevice(deviceIdx); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaSetDevice success, device = %d\n", deviceIdx); + serverEp->curDev_ = deviceIdx; + serverEp->CliDev_ = deviceIdx; + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaSetDevice failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaSetMainDeviceHandle) { + tool::Logging(myName, "CUDA_SET_MAIN_DEVICE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int deviceIdx = reqBuf.Pop(); + serverEp->clientPID_ = reqBuf.Pop(); + serverEp->priority_ = reqBuf.Pop(); + serverEp->recoveryFlag_ = reqBuf.Pop(); + + serverEp->replayFlag_ = serverEp->recoveryFlag_; + cudaError_t exit_code = cudaSetDevice(deviceIdx); + if (exit_code == cudaSuccess) { + tool::Logging(LOG_INFO, serverEp->myName_, "cudaSetMainDevice success, curDev_ = %d, clientPID_ = %d, priority_ = %zu\n", deviceIdx, serverEp->clientPID_, serverEp->priority_); + serverEp->curDev_ = deviceIdx; + serverEp->CliDev_ = deviceIdx; + serverEp->backupFilePath_ = std::string(BACKUP_FILE_DIR) + "flexgv_backup_" + std::to_string(serverEp->clientID_) + "_" + std::to_string(serverEp->clientPID_) + ".dat"; + + // serverEp->SendStatus(123); + + if (serverEp->recoveryFlag_) { + serverEp->LoadFromStorage(); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaSetMainDevice failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaDeviceSynchronizeHandle) { + tool::Logging(myName, "CUDA_DEVICE_SYNCHRONIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaError_t exit_code = cudaDeviceSynchronize(); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaDeviceSynchronize success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaDeviceSynchronize failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtEventCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtEventCallback.cc new file mode 100644 index 0000000..b260b74 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtEventCallback.cc @@ -0,0 +1,161 @@ +/* CUDA Event API */ +#include "../../include/serverEndpoint.h" + +static const char* myName = "CUDARuntimeEventHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudaEventCreateHandle) { + tool::Logging(myName, "CUDA_EVENT_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudaEvent_t event = NULL; + cudaError_t exit_code = cudaEventCreate(&event); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaEventCreate success, event ptr: %p, size=%d\n", event, sizeof(uint64_t)); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, event); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_EVENT_CREATE); + event = (cudaEvent_t)serverEp->GetHandleVirAddr(event, CUDA_EVENT_CREATE); + resBuf.Push64BitPointer(event); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaEventCreate failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaEventCreateWithFlagsHandle) { + tool::Logging(myName, "CUDA_EVENT_CREATE_WITH_FLAGS\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudaEvent_t event = NULL; + unsigned int flags = reqBuf.Pop(); + cudaError_t exit_code = cudaEventCreateWithFlags(&event,flags); + if (exit_code == cudaSuccess) { + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, event); + tool::Logging(LOG_DEBUG, myName, "cudaEventCreateWithFlags success, event ptr: %p(virAddr=%p, pid=%d, ttid=%d), flags=%u\n", event, virAddr, serverEp->clientPID_, reqBuf.GetThreadID(), flags); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_EVENT_CREATE_WITH_FLAGS); + void* virtAddr = (cudaEvent_t)serverEp->GetHandleVirAddr(event, CUDA_EVENT_CREATE_WITH_FLAGS); + resBuf.Push64BitPointer(virtAddr); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + tool::Logging(LOG_DEBUG, myName, "cudaEventCreateWithFlags success, event ptr: %p(new virAddr=%p, pid=%d, ttid=%d), flags=%u\n", event, virtAddr, serverEp->clientPID_, reqBuf.GetThreadID(), flags); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaEventCreateWithFlags failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaEventRecordHandle){ + tool::Logging(myName, "CUDA_EVENT_Record\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t eventVirAddr= reqBuf.Pop(); + cudaEvent_t event = (cudaEvent_t)serverEp->GetHandle(eventVirAddr); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + cudaError_t exit_code = cudaEventRecord(event,stream); + + if (exit_code == cudaSuccess) { +#ifdef GV_HANDLE + Handle_t* handle = serverEp->GetHandleInfo(eventVirAddr); + if (handle != NULL) { + handle->stream = (uint64_t)stream; + } +#endif + + tool::Logging(myName, "cudaEventRecord success, eventPtr=%p, streamPtr=%p(isCommStream=%d)\n",event,stream,stream == serverEp->commStream_); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaEventRecord failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaEventQueryHandle) { + tool::Logging(myName, "CUDA_EVENT_QUERY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t eventVirAddr= reqBuf.Pop(); + cudaEvent_t event = (cudaEvent_t)serverEp->GetHandle(eventVirAddr); + cudaError_t exit_code = cudaSuccess; + if (event == NULL) { + tool::Logging(LOG_ERROR, myName, "cudaEventQuery warning: eventPtr is NULL(virAddr=%p, pid=%d, ttid=%d)\n", eventVirAddr, serverEp->clientPID_, reqBuf.GetThreadID()); + } + else { + exit_code = cudaEventQuery(event); + } + + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_EVENT_QUERY); + resBuf.Push(exit_code); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaEventQuery success, eventPtr=%p\n",event); + } + else if (exit_code == cudaErrorNotReady) { + tool::Logging(myName, "cudaEventQuery success, eventPtr=%p, cudaErrorNotReady(any captured work is incomplete)\n",event); + } + else { + tool::Logging(LOG_ERROR, myName, "cudaEventQuery failed for eventPtr=%p: %s\n", event, cudaGetErrorName(exit_code)); + } + return UCS_OK; +} + +DEFINE_SERVER_AM_CALLBACK(cudaEventDestroyHandle) { + tool::Logging(myName, "CUDA_EVENT_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t eventVirAddr= reqBuf.Pop(); + cudaEvent_t event = (cudaEvent_t)serverEp->GetHandle(eventVirAddr, true); + if (event == NULL) { + tool::Logging(LOG_ERROR, myName, "cudaEventDestroy success, eventPtr=%p(virAddr=%p, pid=%d, ttid=%d)\n",event, eventVirAddr, serverEp->clientPID_, reqBuf.GetThreadID()); + return UCS_OK; + } + cudaError_t exit_code = cudaEventDestroy(event); + if (exit_code == cudaSuccess) { + tool::Logging(LOG_DEBUG, myName, "cudaEventDestroy success, eventPtr=%p(virAddr=%p, pid=%d, ttid=%d)\n",event, eventVirAddr, serverEp->clientPID_, reqBuf.GetThreadID()); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaEventDestroy failed: %s, eventPtr=%p\n", cudaGetErrorName(exit_code), event); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaEventElapsedTimeHandle) { + tool::Logging(myName, "CUDA_EVENT_ELAPSED_TIME\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + float ms = 0; + cudaEvent_t start = (cudaEvent_t)serverEp->GetHandle(reqBuf.Pop()); + cudaEvent_t end = (cudaEvent_t)serverEp->GetHandle(reqBuf.Pop()); + cudaError_t exit_code = cudaEventElapsedTime(&ms,start,end); + if (exit_code == cudaSuccess || exit_code == cudaErrorNotReady) { + tool::Logging(myName, "cudaEventElapsedTime: %s, startPtr=%p, endPtr=%p, ms=%f\n",cudaGetErrorName(exit_code), start,end,ms); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_EVENT_ELAPSED_TIME); + resBuf.Push(exit_code); + resBuf.Push(ms); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaEventElapsedTime failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtExecutionCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtExecutionCallback.cc new file mode 100644 index 0000000..1f30448 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtExecutionCallback.cc @@ -0,0 +1,250 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "CUDARuntimeExecutionHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudaLaunchKernelHandle) { + tool::Logging(myName, "CUDA_LAUNCH_KERNEL\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t func = reqBuf.Pop(); + char* deviceName = reqBuf.AssignCString(); + dim3 gridDim = reqBuf.Pop(); + dim3 blockDim = reqBuf.Pop(); + uint8_t* paraValList = reqBuf.AssignAddrForAll(); + size_t sharedMem = reqBuf.Pop(); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + size_t paramNum = reqBuf.Pop(); + uint16_t* paramOffsets= reqBuf.AssignAddrForAll(); + uint16_t* paramSizes = reqBuf.AssignAddrForAll(); + + CUfunction cuFunc = NULL; + auto it = serverEp->_cuInfoMap.mapHost2CuFunc->find(func); + if (it == serverEp->_cuInfoMap.mapHost2CuFunc->end()) { + tool::Logging(LOG_ERROR, myName, "cudaLaunchKernel failed: unknown function(%p)\n", func); + return UCS_ERR_IO_ERROR; + } + else { + cuFunc = it->second; + } + +#ifdef GV_MEMORY_PTX + if (serverEp->_cuInfoMap.ptxExtractor->_finished == false) { + serverEp->_cuInfoMap.ptxExtractor->_readyClosed = true; + while (serverEp->_cuInfoMap.ptxExtractor->_finished == false) { + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + } + + std::vector>* dev_ptr_list = NULL; + auto newIt = serverEp->_cuInfoMap.mapDevName2DevPtr->find(deviceName); + if (newIt == serverEp->_cuInfoMap.mapDevName2DevPtr->end()) { + tool::Logging(LOG_ERROR, myName, "cudaLaunchKernel failed: unknown kernel name(%s)\n", deviceName); + exit(EXIT_FAILURE); + // return UCS_ERR_IO_ERROR; + } + else { + dev_ptr_list = (std::vector>*)newIt->second; + tool::Logging(myName, "cudaLaunchKernel: kernelName=%s, devPtrListSize=%zu\n", deviceName, dev_ptr_list->size()); + } +#endif // GV_MEMORY_PTX + +#ifdef GV_MEMORY + bool savedDevPtrList = true; + // std::vector>* dev_ptr_list = NULL; + std::set>* dev_ptr_set = NULL; + auto newIt = serverEp->_cuInfoMap.mapDevName2DevPtr->find(deviceName); + if (newIt == serverEp->_cuInfoMap.mapDevName2DevPtr->end()) { + // dev_ptr_list = new std::vector>(); + dev_ptr_set = new std::set>(); + serverEp->_cuInfoMap.mapDevName2DevPtr->insert({deviceName, (uint64_t)dev_ptr_set}); + savedDevPtrList = false; + } + else { + dev_ptr_set = (std::set>*)newIt->second; + tool::Logging(myName, "cudaLaunchKernel: kernelName=%s, devPtrListSize=%zu\n", deviceName, dev_ptr_set->size()); + savedDevPtrList = !(serverEp->_cuInfoMap.isFirstIter); + } +#endif // GV_MEMORY + + void** paramPtrList= (void**)malloc(paramNum * sizeof(void*)); + size_t ptrIdx = 0; + for (size_t i = 0; i < paramNum; i++) { + paramPtrList[i] = paraValList + paramOffsets[i]; + tool::Logging(myName, "param[%zu]: paramPtr=%p, val=%p, paramSize=%d\n", i, (uint64_t*)paramPtrList[i], *(uint64_t*)paramPtrList[i], paramSizes[i]); + +#ifdef GV_MEMORY + if (paramSizes[i] < sizeof(uint64_t)) { + continue; + } + if (!savedDevPtrList) { + size_t paramIntraOffset = 0; + while (paramIntraOffset <= paramSizes[i] - sizeof(uint64_t)){ + uint64_t* paramIntraPtr = (uint64_t*)((uint64_t)paramPtrList[i] + paramIntraOffset); + uint64_t tmpAddr = *paramIntraPtr; + tool::Logging(myName, "\t paramIntraOffset=%zu, tmpAddr=%p\n", paramIntraOffset, (void*)tmpAddr); + uint64_t realPtr = 0; + if (serverEp->_cuInfoMap.blockManager->FindByVirAddr(tmpAddr, realPtr) == -1) { + tool::Logging(myName, "\t warning - tmpAddr(%p) is not in blockManager\n", tmpAddr); + paramIntraOffset ++; + continue; + } + *paramIntraPtr = (uint64_t)realPtr; + tool::Logging(myName, "\t paramIntraOffset=%zu, devPtr=%p\n", paramIntraOffset, *paramIntraPtr); + dev_ptr_set->insert({i, paramIntraOffset}); + paramIntraOffset += sizeof(uint64_t); + } + } + else { + std::vector> dev_ptr_list(dev_ptr_set->begin(), dev_ptr_set->end()); + while (ptrIdx < dev_ptr_list.size()) { + if ((dev_ptr_list)[ptrIdx].first != i) { + break; + } + size_t paramIntraOffset = (dev_ptr_list)[ptrIdx].second; + uint64_t* paramIntraPtr = (uint64_t*)((uint64_t)paramPtrList[i] + paramIntraOffset); + uint64_t usrVirtAddr = *paramIntraPtr; + tool::Logging(LOG_REGS, myName, "\t paramIntraOffset=%zu, usrVirtAddr=%p\n", paramIntraOffset, usrVirtAddr); + uint64_t realPtr = 0; + ptrIdx++; + if (serverEp->_cuInfoMap.blockManager->FindByVirAddr(usrVirtAddr, realPtr) == -1) { + tool::Logging(LOG_REGS, myName, "\t warning - usrVirtAddr(%p) is not in blockManager\n", usrVirtAddr); + continue; + } + *paramIntraPtr = (uint64_t)realPtr; + tool::Logging(myName, "\t paramIntraOffset=%zu, devPtr=%p\n", paramIntraOffset, *paramIntraPtr); + } + } +#endif // GV_MEMORY + +#ifdef GV_MEMORY_PTX + if (strcmp(deviceName, "_ZN2at6native6sbtopk10gatherTopKIfjLi2ELb0EEEvNS_4cuda6detail10TensorInfoIT_T0_EES7_S7_bS7_S7_S8_S7_NS5_IlS7_EES7_PS6_" ) == 0 && i == 0) { + tool::Logging(myName, "param[%zu]: paramPtr=%p, val=%p, paramSize=%d\n", i, (uint64_t*)paramPtrList[i], *(uint64_t*)paramPtrList[i], paramSizes[i]); + size_t paramIntraOffset = 0; + uint64_t* paramIntraPtr = (uint64_t*)((uint64_t)paramPtrList[i] + paramIntraOffset); + uint64_t usrVirtAddr = *paramIntraPtr; // get the devPtr from current param #i + tool::Logging(myName, "\t paramIntraOffset=%zu, usrVirtAddr=%p\n", paramIntraOffset, usrVirtAddr); + if (usrVirtAddr == 0) { // the devPtr can be NULL + continue; + } + uint64_t blockID = GET_BLOCK_ID(usrVirtAddr); + uint64_t blockIntraOffset = GET_BLOCK_INTER_OFFSET(usrVirtAddr); + *paramIntraPtr = (uint64_t)serverEp->GetDevPtr(usrVirtAddr); + tool::Logging(myName, "\t blockID=%p, blockIntraOffset=%zu, devPtr=%p\n", blockID, blockIntraOffset, *paramIntraPtr); + } + + while (ptrIdx < dev_ptr_list->size()) { + if ((*dev_ptr_list)[ptrIdx].first != i) { // next devPtr is not in current param #i + break; + } + size_t paramIntraOffset = (*dev_ptr_list)[ptrIdx].second; + uint64_t* paramIntraPtr = (uint64_t*)((uint64_t)paramPtrList[i] + paramIntraOffset); + uint64_t usrVirtAddr = *paramIntraPtr; // get the devPtr from current param #i + tool::Logging(myName, "\t paramIntraOffset=%zu, usrVirtAddr=%p\n", paramIntraOffset, usrVirtAddr); + ptrIdx++; // ready move to next devPtr, which may be in the same param #i + if (usrVirtAddr == 0) { // the devPtr can be NULL + continue; + } + + uint64_t blockID = GET_BLOCK_ID(usrVirtAddr); + uint64_t blockIntraOffset = GET_BLOCK_INTER_OFFSET(usrVirtAddr); + if (blockID <= 0 || blockID >= serverEp->_cuInfoMap.blockInfoList.size()) { + tool::Logging(myName, "\t warning - blockID(%p), current blockNum=%zu\n", blockID, serverEp->_cuInfoMap.blockInfoList.size()); + // exit(EXIT_FAILURE); + continue; + } + if (blockIntraOffset >= serverEp->_cuInfoMap.blockInfoList[blockID].size) { + tool::Logging(myName, "\t cudaLaunchKernel: warning - blockIntraOffset(%p), current block size=%zu\n", blockIntraOffset, serverEp->_cuInfoMap.blockInfoList[blockID].size); + // exit(EXIT_FAILURE); + continue; + } + *paramIntraPtr = (uint64_t)serverEp->GetDevPtr(usrVirtAddr); + tool::Logging(myName, "\t blockID=%p, blockIntraOffset=%zu, devPtr=%p\n", blockID, blockIntraOffset, *paramIntraPtr); + } +#endif // GV_MEMORY_PTX + + } + + cudaSetDevice(serverEp->curDev_); // explicitly set the device and make sure the context is valid + + CUresult exit_code = cuLaunchKernel(cuFunc, gridDim.x, gridDim.y, gridDim.z, + blockDim.x, blockDim.y, blockDim.z, sharedMem, + (CUstream)stream, paramPtrList, NULL); + if (exit_code == CUDA_SUCCESS) { + tool::Logging(myName, "cudaLaunchKernel success: %s\n", deviceName); +#ifdef GV_MEMORY_PTX + cudaError_t sync_code = cudaDeviceSynchronize(); + if (sync_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaDeviceSynchronize (after launchKernel: %s) failed: %s\n", deviceName, cudaGetErrorName(sync_code)); + exit(EXIT_FAILURE); + } +#endif // GV_MEMORY_PTX + if (serverEp->curIter_ <= 1 // necessary to synchronize the non-blocking stream for the first iterations + || serverEp->recoveryFlag_ // synchronize the replaying kernel (e.g., indexSelectLargeIndex) + ) { + cudaError_t sync_code = cudaStreamSynchronize(stream); + if (sync_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaStreamSynchronize (after launchKernel: %s) failed: %s\n", deviceName, cudaGetErrorName(sync_code)); + exit(EXIT_FAILURE); + } + } + return UCS_OK; + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "cudaLaunchKernel failed: %s\n", errorStr); + // return UCS_ERR_IO_ERROR; + exit(EXIT_FAILURE); + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaFuncGetAttributesHandle) { + tool::Logging(myName, "CUDA_FUNC_GET_ATTRIBUTES\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + struct cudaFuncAttributes attr; + uint64_t func = reqBuf.Pop(); + + CUfunction cuFunc = NULL; + auto it = serverEp->_cuInfoMap.mapHost2CuFunc->find(func); + if (it == serverEp->_cuInfoMap.mapHost2CuFunc->end()) { + tool::Logging(LOG_ERROR, myName, "cudaLaunchKernel failed: unknown function(%p)\n", func); + return UCS_ERR_IO_ERROR; + } + else { + cuFunc = it->second; + } + + int exit_code = CUDA_SUCCESS; + int shrSzBytes = 0; + int conSzBytes = 0; + int locSzBytes = 0; + exit_code |= cuFuncGetAttribute(&shrSzBytes, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, cuFunc); + exit_code |= cuFuncGetAttribute(&conSzBytes, CU_FUNC_ATTRIBUTE_CONST_SIZE_BYTES, cuFunc); + exit_code |= cuFuncGetAttribute(&locSzBytes, CU_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, cuFunc); + attr.sharedSizeBytes = shrSzBytes; + attr.constSizeBytes = conSzBytes; + attr.localSizeBytes = locSzBytes; + exit_code |= cuFuncGetAttribute((int*)&attr.maxThreadsPerBlock, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, cuFunc); + exit_code |= cuFuncGetAttribute((int*)&attr.numRegs, CU_FUNC_ATTRIBUTE_NUM_REGS, cuFunc); + exit_code |= cuFuncGetAttribute((int*)&attr.ptxVersion, CU_FUNC_ATTRIBUTE_PTX_VERSION, cuFunc); + exit_code |= cuFuncGetAttribute((int*)&attr.binaryVersion, CU_FUNC_ATTRIBUTE_BINARY_VERSION, cuFunc); + exit_code |= cuFuncGetAttribute((int*)&attr.cacheModeCA, CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, cuFunc); + exit_code |= cuFuncGetAttribute((int*)&attr.maxDynamicSharedSizeBytes, CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, cuFunc); + exit_code |= cuFuncGetAttribute((int*)&attr.preferredShmemCarveout, CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, cuFunc); + + + if (exit_code == CUDA_SUCCESS) { + tool::Logging(myName, "cudaFuncGetAttributes success\n"); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_FUNC_GET_ATTRIBUTES); + resBuf.Push(attr); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaFuncGetAttributes failed\n"); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtInternalCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtInternalCallback.cc new file mode 100644 index 0000000..5fc7dfd --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtInternalCallback.cc @@ -0,0 +1,319 @@ +#include "../../include/serverEndpoint.h" + +#include "../../include/hook/fatBinary.h" + +extern "C" { +extern void **__cudaRegisterFatBinary(void *fatCubin); +extern void __cudaRegisterFatBinaryEnd(void **fatCubinHandle); +extern void __cudaUnregisterFatBinary(void **fatCubinHandle); +extern void __cudaRegisterFunction(void **fatCubinHandle, const char *hostFun, + char *deviceFun, const char *deviceName, + int thread_limit, uint3 *tid, uint3 *bid, + dim3 *bDim, dim3 *gDim, int *wSize); +extern void __cudaRegisterVar(void **fatCubinHandle, char *hostVar, + char *deviceAddress, const char *deviceName, + int ext, size_t size, int constant, int global); +extern void __cudaRegisterSharedVar(void **fatCubinHandle, void **devicePtr, + size_t size, size_t alignment, int storage); +extern void __cudaRegisterShared(void **fatCubinHandle, void **devicePtr); +// extern void __cudaRegisterTexture(void **fatCubinHandle, +// const textureReference *hostVar, +// void **deviceAddress, char *deviceName, +// int dim, int norm, int ext); +// extern void __cudaRegisterSurface(void **fatCubinHandle, +// const surfaceReference *hostVar, +// void **deviceAddress, char *deviceName, +// int dim, int ext); +} + +static const char* myName = "CUDARuntimeInternalHandle"; + +DEFINE_SERVER_AM_CALLBACK(__cudaRegisterHandle) { + tool::Logging(LOG_REGS, myName, "__CUDA_REGISTER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RegisterIOV reqBuf = RegisterIOV(header, header_length, data); + size_t reqNum = reqBuf.GetRequestNum(); + CUmodule curModule = NULL; + + if (!serverEp->initFlag_) { + cudaSetDevice(serverEp->curDev_); + cudaFree(0); // explicitly initialize the context for the current device + + int streamPriority = -1 * (int)serverEp->priority_; + cudaError_t exit_code = cudaStreamCreateWithPriority(&serverEp->defaultStream_, cudaStreamNonBlocking, streamPriority); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaStreamCreate failed: %s\n", cudaGetErrorString(exit_code)); + exit(EXIT_FAILURE); + } + else { + tool::Logging(LOG_REGS, myName, "serverEp defaultStream = %p\n", serverEp->defaultStream_); + } + // serverEp->defaultStream_ = NULL; + + tool::Logging(myName, "initFlag_ is false\n"); + for (int i = 0; i < 3; i++) { + serverEp->shmQueSizes_[i] = new SharedMemoryOpt(("/flexgv_shm_"+std::to_string(serverEp->clientID_)+ "_" + std::to_string(serverEp->clientPID_) + "_" + std::to_string(i + 1)).c_str(), sizeof(size_t) * 3, false); // dataFeeder may be the first one to create the shared memory and initialize the space + } + +#ifdef GV_MEMORY_PTX + serverEp->_cuInfoMap.imageQueue = new moodycamel::BlockingReaderWriterQueue< std::pair >(240); + serverEp->_cuInfoMap.ptxExtractor = new PTXExtractor(serverEp->_cuInfoMap.imageQueue, serverEp->_cuInfoMap.mapDevName2DevPtr); + boost::thread_attributes attrs; + attrs.set_stack_size(THREAD_STACK_SIZE); + serverEp->_ptxThread = new boost::thread(attrs, boost::bind(&PTXExtractor::Run, serverEp->_cuInfoMap.ptxExtractor)); +#endif // GV_MEMORY_PTX + + serverEp->initFlag_ = true; + } + else { + tool::Logging(myName, "initFlag_ is true\n"); + } + + for (size_t i = 0; i < reqNum; i++) { + int reqType = reqBuf.Pop(); + switch (reqType) { + case __CUDA_REGISTER_FAT_BINARY: { + uint64_t fatBinHandle = reqBuf.Pop(); + size_t fatBinSize = reqBuf.Pop(); + size_t kernelNum = reqBuf.Pop(); + void* fatBinText = reqBuf.AssignAddr(); + void* image = fatBinText; +#ifdef GV_MEMORY_PTX + image = malloc(fatBinSize); + memcpy(image, fatBinText, fatBinSize); + serverEp->_fatbinList.push_back(image); + + if (serverEp->_cuInfoMap.imageQueue){ + serverEp->_cuInfoMap.imageQueue->enqueue(std::make_pair(image, fatBinSize)); + } +#endif // GV_MEMORY_PTX + + CUresult exit_code = cuModuleLoadData(&curModule, image); + if (exit_code == CUDA_SUCCESS) { + serverEp->_cuInfoMap.mapFatBinHandle2CuModule->insert({fatBinHandle, curModule}); + tool::Logging(LOG_REGS, myName, "__cudaRegisterFatBinary success: loaded the module(%p) from fatCubin(%p) text\n", curModule, fatBinHandle); + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "__cudaRegisterFatBinary failed: %s\n", errorStr); + return UCS_ERR_IO_ERROR; + } + break; + } + case __CUDA_REGISTER_FUNCTION: { + uint64_t fatBinHandler = reqBuf.Pop(); + const char* hostFun = (const char *)reqBuf.Pop(); + const char* deviceName = strdup(reqBuf.AssignCString()); + size_t paramNum = reqBuf.Pop(); + CUfunction cuFunc = NULL; + CUresult exit_code = cuModuleGetFunction(&cuFunc, curModule, deviceName); + if (exit_code == CUDA_SUCCESS) { + serverEp->_cuInfoMap.mapHost2CuFunc->insert({(uint64_t)hostFun, cuFunc}); + tool::Logging(LOG_REGS, myName, "__cudaRegisterFunction success: registered function(%s) with %zu parameters\n", deviceName, paramNum); + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "__cudaRegisterFunction failed for function(%s): %s\n", deviceName, errorStr); + return UCS_ERR_IO_ERROR; + } + break; + } + case __CUDA_REGISTER_VAR: { + uint64_t fatBinHandler = reqBuf.Pop(); + char* hostVar = (char*)reqBuf.Pop(); + const char* deviceName = strdup(reqBuf.AssignCString()); + CUdeviceptr devPtr = 0; + size_t bytes = 0; + CUresult exit_code = cuModuleGetGlobal(&devPtr, &bytes, curModule, deviceName); + if (exit_code == CUDA_SUCCESS) { + serverEp->_cuInfoMap.mapHostVar2CuDevPtr->insert({(uint64_t)hostVar, devPtr}); + tool::Logging(LOG_REGS, myName, "__cudaRegisterVar success: registered variable(hostVar: %p, devName: %s, devPtr: %p)\n", hostVar, deviceName, devPtr); + } + else { + const char* errorStr; + tool::Logging(LOG_ERROR, myName, "__cudaRegisterVar failed for variable(%s): %s\n", deviceName, errorStr); + return UCS_ERR_IO_ERROR; + } + break; + } + } + } + tool::Logging(LOG_DEBUG, myName, "__cudaRegister success: registered %zu items, %zu bytes in total\n", reqNum, length); + return UCS_OK; +} + +DEFINE_SERVER_AM_CALLBACK(__cudaRegisterFatBinaryHandle){ + tool::Logging(LOG_REGS, myName, "__CUDA_REGISTER_FAT_BINARY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t fatBinHandle= reqBuf.Pop(); + size_t fatBinSize = reqBuf.Pop(); + size_t kernelNum = reqBuf.Pop(); + void* fatBinText = reqBuf.AssignAddr(); + void* image = NULL; + image = malloc(fatBinSize); + memcpy(image, fatBinText, fatBinSize); + +#ifdef GV_MEMORY_PTX + if (serverEp->_cuInfoMap.imageQueue){ + serverEp->_cuInfoMap.imageQueue->enqueue(std::make_pair(image, fatBinSize)); + } +#endif // GV_MEMORY_PTX + + cudaSetDevice(serverEp->curDev_); + + + cudaFree(0); // explicitly initialize the context for device #i + + CUmodule cuModule = NULL; + CUresult exit_code = cuModuleLoadData(&cuModule, image); + if (exit_code == CUDA_SUCCESS) { + tool::Logging(LOG_REGS, myName, "__cudaRegisterFatBinary success: loaded the module(%p) from fatCubin(%p) text in device #%d\n", cuModule, fatBinHandle, serverEp->curDev_); + serverEp->_cuInfoMap.mapFatBinHandle2CuModule->insert({fatBinHandle, cuModule}); + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "__cudaRegisterFatBinary failed in device #%d: %s\n", serverEp->curDev_, errorStr); + return UCS_ERR_IO_ERROR; + } + + // delete[] fatBinText; + cudaSetDevice(serverEp->curDev_); + + if (param->recv_attr & UCP_AM_RECV_ATTR_FLAG_RNDV) { + free(image); + } + + return UCS_OK; +} + +DEFINE_SERVER_AM_CALLBACK(__cudaRegisterFunctionHandle) { + tool::Logging(LOG_REGS, myName, "__CUDA_REGISTER_FUNCTION\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t fatBinHandler = reqBuf.Pop(); + const char* hostFun = (const char *)reqBuf.Pop(); + const char* deviceName = strdup(reqBuf.AssignCString()); + size_t paramNum = reqBuf.Pop(); + + cudaSetDevice(serverEp->curDev_); + auto it = serverEp->_cuInfoMap.mapFatBinHandle2CuModule->find(fatBinHandler); + if (it == serverEp->_cuInfoMap.mapFatBinHandle2CuModule->end()) { + tool::Logging(LOG_ERROR, myName, "__cudaRegisterFunction failed: unknown handler(%p) in device #%d\n", fatBinHandler, serverEp->curDev_); + return UCS_ERR_IO_ERROR; + } + CUmodule cuModule = it->second; + CUfunction cuFunc = NULL; + CUresult exit_code = cuModuleGetFunction(&cuFunc, cuModule, deviceName); + if (exit_code == CUDA_SUCCESS) { + serverEp->_cuInfoMap.mapHost2CuFunc->insert({(uint64_t)hostFun, cuFunc}); + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "__cudaRegisterFunction failed for function(%s) in device #%d: %s\n", deviceName, serverEp->curDev_, errorStr); + return UCS_ERR_IO_ERROR; + } + + tool::Logging(LOG_REGS, myName, "__cudaRegisterFunction success: registered function(%s) with %zu parameters\n", deviceName, paramNum); + cudaSetDevice(serverEp->curDev_); + return UCS_OK; +} + +void ServerEndpoint::__cudaUnregisterFatBinaryHandle() { + tool::Logging(myName, "__CUDA_UNREGISTER_FAT_BINARY\n"); + + for (auto& pair : *_cuInfoMap.mapFatBinHandle2CuModule) { + CUresult exit_code = cuModuleUnload(pair.second); + if (exit_code != CUDA_SUCCESS) { + tool::Logging(LOG_ERROR, myName, "__cudaUnregisterFatBinary failed: can not unload the module(%p) from fatCubin(%p) text\n", pair.second, pair.first); + } + } + for (int i = 0; i < _fatbinList.size(); i++) { + free(_fatbinList[i]); + } + for (auto& pair : *_cuInfoMap.mapDevName2DevPtr) { + auto tmpSet = reinterpret_cast>*>(pair.second); + delete tmpSet; + } + _cuInfoMap.mapFatBinHandle2CuModule->clear(); + _cuInfoMap.mapHost2CuFunc->clear(); + _cuInfoMap.mapHostVar2CuDevPtr->clear(); + +#ifdef GV_MEMORY + const std::vector& blocks = _cuInfoMap.blockManager->GetBlocks(); + for (int i = 0; i < blocks.size(); i++) { + if (blocks[i].valid == false) { // maybe have already been freed + continue; + } + cudaError_t exit_code = cudaFree((void*)blocks[i].devPtr); + _cuInfoMap.blockManager->ResetBlock(0, i); + } + delete _cuInfoMap.blockManager; +#else + for (size_t blockIdx = 0; blockIdx < _cuInfoMap.blockInfoList.size(); blockIdx++) { + if (_cuInfoMap.blockInfoList[blockIdx].valid == false) { + continue; + } + cudaError_t exit_code = cudaFree((void*)_cuInfoMap.blockInfoList[blockIdx].devPtr); // maybe have already been freed + _cuInfoMap.blockInfoList[blockIdx].valid = false; + } +#endif + + tool::Logging(LOG_DEBUG, myName, "__cudaUnregisterFatBinary success: unloaded all the modules and cleared all the maps of client #%llu\n", clientID_); + // tool::Logging(LOG_INFO, myName, "blockInfoList size: %zu, handleInfoList size: %zu (valid size: %zu)\n", _cuInfoMap.blockInfoList.size(), _cuInfoMap.handleManager->GetCapacity(), _cuInfoMap.handleManager->GetValidHandleNum()); + + cudaStreamDestroy(defaultStream_); + +#ifdef GV_BACKUP + StopBackup(); + StopCommEventMonitor(); +#endif // GV_BACKUP + +#ifdef GV_HANDLE + delete _cuInfoMap.handleManager; +#endif // GV_HANDLE + + delete _cuInfoMap.mapFatBinHandle2CuModule; + delete _cuInfoMap.mapHost2CuFunc; + delete _cuInfoMap.mapHostVar2CuDevPtr; + + delete _cuInfoMap.ptxExtractor; // ptxExtractor will clean up the imageQueue, mapDevName2DevPtr + delete _ptxThread; +} + +DEFINE_SERVER_AM_CALLBACK(__cudaRegisterVarHandle) { + tool::Logging(LOG_REGS, myName, "__CUDA_REGISTER_VAR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t fatBinHandler = reqBuf.Pop(); + + char* hostVar = (char*)reqBuf.Pop(); + const char* deviceName = strdup(reqBuf.AssignCString()); + + cudaSetDevice(serverEp->curDev_); + + auto it = serverEp->_cuInfoMap.mapFatBinHandle2CuModule->find(fatBinHandler); + if (it == serverEp->_cuInfoMap.mapFatBinHandle2CuModule->end()) { + tool::Logging(LOG_ERROR, myName, "__cudaRegisterVar failed: unknown handler(%p) in device #%d\n", fatBinHandler, serverEp->curDev_); + return UCS_ERR_IO_ERROR; + } + CUmodule cuModule = it->second; + CUdeviceptr devPtr = 0; + size_t bytes = 0; + CUresult exit_code = cuModuleGetGlobal(&devPtr, &bytes, cuModule, deviceName); + if (exit_code == CUDA_SUCCESS) { + tool::Logging(LOG_REGS, myName, "__cudaRegisterVar success: registered variable(hostVar: %p, devName: %s, devPtr: %p in device #%d)\n", hostVar, deviceName, devPtr, serverEp->curDev_); + serverEp->_cuInfoMap.mapHostVar2CuDevPtr->insert({(uint64_t)hostVar, devPtr}); + } + else { + tool::Logging(LOG_ERROR, myName, "__cudaRegisterVar failed: can not load the variable in device #%d\n", serverEp->curDev_); + return UCS_ERR_IO_ERROR; + } + + cudaSetDevice(serverEp->curDev_); + return UCS_OK; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtMemoryCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtMemoryCallback.cc new file mode 100644 index 0000000..00f1bb5 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtMemoryCallback.cc @@ -0,0 +1,467 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "CUDARuntimeMemoryHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudaMallocHandle){ + tool::Logging(myName, "CUDA_MALLOC\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + size_t cuda_size = reqBuf.Pop(); + bool essential = reqBuf.Pop(); + + void* devPtr = NULL; + cudaError_t exit_code = cudaMalloc(&devPtr, cuda_size); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMalloc success, devPtr = %p with size = %d\n", devPtr, cuda_size); + uint64_t virtAddr = (uint64_t)devPtr; + +#ifdef GV_MEMORY + virtAddr = serverEp->_cuInfoMap.blockManager->AddBlock((uint64_t)devPtr, cuda_size, essential); +#else + size_t blockNum = ( + serverEp->_cuInfoMap.blockInfoList.emplace_back(Block_t{.devPtr = (uint64_t)devPtr, .size = cuda_size, .valid = true}), + serverEp->_cuInfoMap.blockInfoList.size() - 1 + ); +#endif // GV_MEMORY + +#ifdef GV_MEMORY_PTX + if (blockNum >= BLOCKS_MAX_NUM) { + tool::Logging(LOG_ERROR, myName, "cudaMalloc failed: current blockNum(%zu) >= BLOCKS_MAX_NUM(%zu)\n", blockNum, BLOCKS_MAX_NUM); + return UCS_ERR_IO_ERROR; + } + virtAddr = blockNum << BLOCK_SHIFT_BIT; +#endif // GV_MEMORY_PTX + +#ifdef GV_Scheduler + serverEp->sche->cal_job_mem(cuda_size); +#endif // GV_Scheduler + + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_MALLOC); + // resBuf.Push64BitPointer(devPtr); + resBuf.Push(virtAddr); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMalloc failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemsetHandle) { + tool::Logging(myName, "CUDA_MEMSET\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + void* devPtr = serverEp->GetDevPtr(reqBuf.Pop()); + // void* devPtr = (void*)reqBuf->Pop(); + int value = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + + cudaError_t exit_code = cudaMemset(devPtr, value, count); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemset success, devPtr = %p(%llu) with value = %d and count = %d\n", devPtr, devPtr, value, count); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemset failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemsetAsyncHandle) { + tool::Logging(myName, "CUDA_MEMSET_ASYNC\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + void* devPtr = serverEp->GetDevPtr(reqBuf.Pop()); + // void* devPtr = (void*)reqBuf->Pop(); + int value = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + + cudaError_t exit_code = cudaMemsetAsync(devPtr, value, count, stream); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemset success, devPtr = %p(%llu) with value = %d , count = %d , streamPtr=%p\n", devPtr, devPtr, value, count, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemset failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemGetInfoHandle) { + tool::Logging(myName, "CUDA_MEM_GET_INFO\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int replayDone = reqBuf.Pop(); + size_t free = 0; + size_t total = 0; + cudaError exit_code = cudaMemGetInfo(&free,&total); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemGetInfo success, free = %lu, total= %lu\n",free,total); + if (replayDone) { + serverEp->recoveryFlag_ = false; + serverEp->_cuInfoMap.handleManager->Shrink(); + tool::Logging(LOG_INFO, "Backup", "replay done.\n"); + } + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_MEM_GET_INFO); + resBuf.Push(free); + resBuf.Push(total); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemGetInfo failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaFreeHandle) { + tool::Logging(myName, "CUDA_FREE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virtAddr = reqBuf.Pop(); + void* devPtr = serverEp->GetDevPtr(virtAddr); + + cudaError_t exit_code = cudaFree(devPtr); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaFree success, devPtr = %p\n", devPtr); +#ifdef GV_MEMORY_PTX + size_t blockIdx = serverEp->GetBlockIdx(virtAddr); + serverEp->_cuInfoMap.blockInfoList[blockIdx].valid = false; +#endif // GV_MEMORY_PTX + +#ifdef GV_MEMORY + serverEp->_cuInfoMap.blockManager->ResetBlock(virtAddr); +#endif // GV_MEMORY + + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaFree failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyH2DHandle) { + tool::Logging(myName, "CUDA_MEMCPY_H2D\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + cudaMemcpyKind kind = cudaMemcpyHostToDevice; + size_t count = 0; + void* dst = NULL; + ucs_status_t status = UCS_OK; + uint8_t* headers = (uint8_t*)header; // header contains parameters for cudaMemcpy H2D + memcpy(&kind, headers, sizeof(cudaMemcpyKind)); + memcpy(&count, headers + sizeof(cudaMemcpyKind), sizeof(size_t)); + memcpy(&dst, headers + sizeof(cudaMemcpyKind) + sizeof(size_t), sizeof(uint64_t)); + dst = serverEp->GetDevPtr((uint64_t)dst); + + if (kind != cudaMemcpyHostToDevice || count != length) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpy H2D failed: invalid cudaMemcpyKind(%d) or count(%zu)\n", kind, count); + return UCS_ERR_IO_ERROR; + } + if (param->recv_attr & UCP_AM_RECV_ATTR_FLAG_RNDV) { + tool::Logging(myName, "cudaMemcpyH2D: RNDV request.\n"); + Request_t recv_request_ctx = {.type = 1, .complete = 0}; + ucp_request_param_t param2; + param2.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | + UCP_OP_ATTR_FIELD_DATATYPE | + UCP_OP_ATTR_FIELD_USER_DATA| + UCP_OP_ATTR_FIELD_MEMORY_TYPE; + param2.op_attr_mask |= UCP_OP_ATTR_FLAG_NO_IMM_CMPL; + param2.datatype = ucp_dt_make_contig(1); + param2.user_data = &recv_request_ctx; + param2.cb.recv_am = (ucp_am_recv_data_nbx_callback_t)RecvCallBack; + param2.memory_type = UCS_MEMORY_TYPE_CUDA; + Request_t* rndv_request = (Request_t*)ucp_am_recv_data_nbx(serverEp->_dataWorker, data, dst, count, ¶m2); + status = Wait(rndv_request, &recv_request_ctx, &serverEp->_dataWorker); + if (status == UCS_OK) { + tool::Logging(myName, "cudaMemcpy H2D: Host -> Device(%p) with %zu B\n", dst, count); + } + else { + tool::Logging(myName, "cudaMemcpy H2D failed: ucp_am_recv_data_nbx(%s)\n", ucs_status_string(status)); + } + ucp_request_free(rndv_request); + return status; + } + else { + tool::Logging(myName, "cudaMemcpy H2D: Eager request.\n"); + cudaError_t exit_code = cudaMemcpy(dst, data, count, kind); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemcpy H2D: Host -> Device(%p) with %zu B\n", dst, count); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpy H2D failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyD2HHandle) { + tool::Logging(myName, "CUDA_MEMCPY_D2H\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaMemcpyKind kind = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + void* src = serverEp->GetDevPtr(reqBuf.Pop()); + + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_MEMCPY_D2H); + resBuf.Push((uint8_t*)src, count); + serverEp->SendResponse(&resBuf, ¶m->reply_ep, UCS_MEMORY_TYPE_CUDA); + + tool::Logging(myName, "cudaMemcpy D2H: Device(%p) -> Host with %zu B\n", src, count); + return UCS_OK; +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyD2DHandle) { + tool::Logging(myName, "CUDA_MEMCPY_D2D\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaMemcpyKind kind = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + void* src = serverEp->GetDevPtr(reqBuf.Pop()); + void* dst = serverEp->GetDevPtr(reqBuf.Pop()); + + cudaError_t exit_code = cudaMemcpy(dst, src, count, kind); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemcpy D2D: Device(%p) -> Device(%p) with %zu B\n", src, dst, count); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpy D2D failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyAsyncH2DHandle) { + tool::Logging(myName, "CUDA_MEMCPY_ASYNC_H2D\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + cudaMemcpyKind kind = cudaMemcpyHostToDevice; + size_t count = 0; + cudaStream_t stream = NULL; + uint8_t CurType = MEMCPY_OTHER; + void* dst = NULL; + ucs_status_t status = UCS_OK; + uint8_t* headers = (uint8_t*)header; // header contains parameters for cudaMemcpyAsync H2D + size_t headerOffset= 0; + memcpy(&kind, headers + headerOffset, sizeof(cudaMemcpyKind)); + headerOffset += sizeof(cudaMemcpyKind); + memcpy(&count, headers + headerOffset, sizeof(size_t)); + headerOffset += sizeof(size_t); + memcpy(&stream,headers + headerOffset, sizeof(uint64_t)); + stream = (cudaStream_t)serverEp->GetHandle((uint64_t)stream); + headerOffset += sizeof(uint64_t); + memcpy(&CurType,headers + headerOffset, sizeof(uint8_t)); + headerOffset += sizeof(uint8_t); + memcpy(&dst, headers + headerOffset, sizeof(uint64_t)); + headerOffset += sizeof(uint64_t); + dst = serverEp->GetDevPtr((uint64_t)dst); + + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + + if (kind != cudaMemcpyHostToDevice) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync H2D failed: invalid cudaMemcpyKind(%d)\n", kind); + return UCS_ERR_IO_ERROR; + } + + if (CurType != MEMCPY_OTHER) { // with data preloading optimization, data is already in shared memory queue + size_t shmQueueSize= 0; + size_t shmQueueIden= 0; + if (serverEp->shmQueues_[CurType - 1] == NULL) { + if (serverEp->shmQueSizes_[CurType - 1]->ReadCurSize(&shmQueueSize) == false || shmQueueSize == 0) { + tool::Logging(LOG_ERROR, myName, "failed: read shared memory queue size failed\n"); + // return UCS_ERR_IO_ERROR; + exit(EXIT_FAILURE); + } +#ifdef GV_Scheduler + if(serverEp->shmQueSizes_[CurType - 1]->ReadCurNumIt(&(serverEp->numIterations)) == false){ + tool::Logging(LOG_ERROR, myName, "failed: read numIterations failed\n", kind); + return UCS_ERR_IO_ERROR; + } + serverEp->sche->get_Iteration(serverEp->numIterations); + tool::Logging(LOG_INFO, myName, "numIterations=%d\n", serverEp->numIterations); +#endif + if (serverEp->shmQueSizes_[CurType - 1]->ReadCurIdent(&shmQueueIden) == false) { + tool::Logging(LOG_ERROR, myName, "failed: read shared memory queue identify prefix failed\n"); + // return UCS_ERR_IO_ERROR; + exit(EXIT_FAILURE); + } + tool::Logging(myName, "cudaMemcpyAsync H2D: shmQueueSize=%zu, shmQueueIdentifyPrefix=%zu\n", shmQueueSize, shmQueueIden); + serverEp->shmQueues_[CurType - 1] = CMessageQueue::GetInstance(GENERATE_KEY(shmQueueIden, CurType), shmQueueSize, eQueueModel::ONE_READ_ONE_WRITE); + } + if (serverEp->lastCopyLen_[CurType] != 0 && serverEp->lastCopyType_ == CurType) { // avoid pop the empty queue + serverEp->shmQueues_[CurType - 1]->Pop(serverEp->lastCopyLen_[CurType]); + } + uint8_t* src = NULL; + size_t len = 0; + int error_code = serverEp->shmQueues_[CurType - 1]->ReadMessage(&src, &len); //todo + tool::Logging(myName,"ZWX: kind=%d CurType=%02x error_code=%d len=%zu count=%zu last_copyLen=%zu dst=%p\n", kind, CurType, error_code, len, count, serverEp->lastCopyLen_[CurType], dst); + while(error_code != 0){ // blocking read until data is ready in shared memory queue + // tool::Logging(myName, "HYF# repeated reading"); + // tool::Logging(myName, "read failed from shared memory queue: %zu\n", len); + error_code = serverEp->shmQueues_[CurType - 1]->ReadMessage(&src, &len); + } + if (len == count) { + tool::Logging(myName, "cudaMemcpyAsync H2D: read data (%zu B) from shm queue(type=%02x)\n", len, CurType); + // printf("data[0-7]: "); + // for (size_t i = 0; i < 8; i++) { + // printf("%02x ", src[i]); + // } + // printf(", data[%zu-%zu]: ", len - 8, len - 1); + // for (size_t i = len - 8; i < len; i++) { + // printf("%02x ", src[i]); + // } + // printf("\n"); + + cudaError_t exit_code = cudaMemcpyAsync(dst, src, len, kind, stream); + if (exit_code == cudaSuccess) { + //cudaStreamSynchronize(stream); // make sure the data has been copied completely + serverEp->lastCopyLen_[CurType] = len; + serverEp->lastCopyType_ = CurType; + tool::Logging(myName, "cudaMemcpyAsync H2D: Host(%p) -> Device(%p) with %zu B\n", src, dst, count); +#ifdef GV_Scheduler + ++serverEp->numIterations; + if((serverEp->numIterations & 1) == 0) { + serverEp->sche->cal_add_It(serverEp->clientID_); + } +#endif + // return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync H2D failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync H2D failed: read failed from shm queue, len(%zu B) != cnt(%zu B)\n", len, count); + return UCS_ERR_IO_ERROR; + } + } + else { // without data preloading, data transfer from remote host to current device + if (count != length) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync H2D failed: invalid count(%zu), recv length(%zu)\n", count, length); + return UCS_ERR_IO_ERROR; + } + + if (param->recv_attr & UCP_AM_RECV_ATTR_FLAG_RNDV) { + tool::Logging(myName, "cudaMemcpyAsync H2D: RNDV request.\n"); + Request_t recv_request_ctx = {.type = 1, .complete = 0}; + ucp_request_param_t param2; + param2.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | + UCP_OP_ATTR_FIELD_DATATYPE | + UCP_OP_ATTR_FIELD_USER_DATA| + UCP_OP_ATTR_FIELD_MEMORY_TYPE; + param2.op_attr_mask |= UCP_OP_ATTR_FLAG_NO_IMM_CMPL; + param2.datatype = ucp_dt_make_contig(1); + param2.user_data = &recv_request_ctx; + param2.cb.recv_am = (ucp_am_recv_data_nbx_callback_t)RecvCallBack; + param2.memory_type = UCS_MEMORY_TYPE_CUDA; + Request_t* rndv_request = (Request_t*)ucp_am_recv_data_nbx(serverEp->_dataWorker, data, dst, count, ¶m2); + status = Wait(rndv_request, &recv_request_ctx, &serverEp->_dataWorker); + ucp_request_free(rndv_request); + if (status == UCS_OK) { + tool::Logging(myName, "cudaMemcpyAsync H2D: Host -> Device(%p) with %zu B\n", dst, count); + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync H2D failed: ucp_am_recv_data_nbx(%s)\n", ucs_status_string(status)); + return status; + } + // return status; + } + else { + tool::Logging(myName, "cudaMemcpyH2D: Eager request.\n"); + cudaError_t exit_code = cudaMemcpyAsync(dst, data, count, kind, stream); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemcpyAsync H2D: Host -> Device(%p) with %zu B\n", dst, count); + // return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyH2D failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } + } + } + return UCS_OK; +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyAsyncD2HHandle) { + tool::Logging(myName, "CUDA_MEMCPY_ASYNC_D2H\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaMemcpyKind kind = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + void* src = serverEp->GetDevPtr(reqBuf.Pop()); + + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_MEMCPY_ASYNC_D2H); + resBuf.Push((uint8_t*)src, count); + serverEp->SendResponse(&resBuf, ¶m->reply_ep, UCS_MEMORY_TYPE_CUDA); + + tool::Logging(myName, "cudaMemcpyAsync D2H: Device(%p) -> Host with %zu B\n", src, count); + return UCS_OK; +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyAsyncD2DHandle) { + tool::Logging(myName, "CUDA_MEMCPY_ASYNC_D2D\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaMemcpyKind kind = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + void* src = serverEp->GetDevPtr(reqBuf.Pop()); + void* dst = serverEp->GetDevPtr(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + cudaError_t exit_code = cudaMemcpyAsync(dst, src, count, kind, stream); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemcpyAsync D2D: Device(%p) -> Device(%p) with %zu B\n", src, dst, count); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync D2D failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyToSymbolHandle) { + tool::Logging(myName, "CUDA_MEMCPY_TO_SYMBOL\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaMemcpyKind kind = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + size_t offset = reqBuf.Pop(); + + if (kind == cudaMemcpyHostToDevice || kind == cudaMemcpyDefault) { + uint64_t symbol = reqBuf.Pop(); + void* src = reqBuf.AssignAddrForAll(); + + + auto it = serverEp->_cuInfoMap.mapHostVar2CuDevPtr->find(symbol); + if (it == serverEp->_cuInfoMap.mapHostVar2CuDevPtr->end()) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyToSymbolHandle failed: unknown symbol(%p)\n", symbol); + return UCS_ERR_IO_ERROR; + } + void* cuDevPtr = (void*)it->second; + // cudaError_t exit_code = cudaMemcpyToSymbol((void*)cuDevPtr, src, count, offset, kind); + cudaError_t exit_code = cudaMemcpy((uint8_t*)cuDevPtr + offset, src, count, kind); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemcpyToSymbol success\n"); + return UCS_OK; + } + else { + tool::Logging(myName, "cudaMemcpyToSymbol failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyToSymbol failed: Unknown cudaMemcpyKind: %d\n", kind); + return UCS_ERR_IO_ERROR; + } +} + + diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtOtherCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtOtherCallback.cc new file mode 100644 index 0000000..c2a8ef0 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtOtherCallback.cc @@ -0,0 +1,80 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "CUDARuntimeOtherHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudaOccupancyMaxActiveBlocksPerMultiprocessorHandle) { + tool::Logging(myName, "CUDA_OCCUPANCY_MAX_ACTIVE_BLOCKS_PER_MULTIPROCESSOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t hostFun = reqBuf.Pop(); + int blockSize = reqBuf.Pop(); + size_t dynSMemSize = reqBuf.Pop(); + + CUfunction cuFunc = NULL; + auto it = serverEp->_cuInfoMap.mapHost2CuFunc->find(hostFun); + if (it == serverEp->_cuInfoMap.mapHost2CuFunc->end()) { + tool::Logging(LOG_ERROR, myName, "cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags failed: unknown function(%p)\n", hostFun); + return UCS_ERR_IO_ERROR; + } + else { + cuFunc = it->second; + } + + cudaSetDevice(serverEp->curDev_); + + int numBlocks = 0; + CUresult exit_code = cuOccupancyMaxActiveBlocksPerMultiprocessor(&numBlocks, cuFunc, blockSize, dynSMemSize); + if (exit_code == CUDA_SUCCESS) { + tool::Logging(myName, "cudaOccupancyMaxActiveBlocksPerMultiprocessor success, numBlocks = %d\n", numBlocks); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_OCCUPANCY_MAX_ACTIVE_BLOCKS_PER_MULTIPROCESSOR); + resBuf.Push(numBlocks); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "cudaOccupancyMaxActiveBlocksPerMultiprocessor failed: %s\n", errorStr); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlagsHandle) { + tool::Logging(myName, "CUDA_OCCUPANCY_MAX_ACTIVE_BLOCKS_PER_MULTIPROCESSOR_WITH_FLAGS\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t hostFun = reqBuf.Pop(); + int blockSize = reqBuf.Pop(); + size_t dynSMemSize = reqBuf.Pop(); + unsigned int flags = reqBuf.Pop(); + + CUfunction cuFunc = NULL; + auto it = serverEp->_cuInfoMap.mapHost2CuFunc->find(hostFun); + if (it == serverEp->_cuInfoMap.mapHost2CuFunc->end()) { + tool::Logging(LOG_ERROR, myName, "cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags failed: unknown function(%p)\n", hostFun); + return UCS_ERR_IO_ERROR; + } + else { + cuFunc = it->second; + } + + cudaSetDevice(serverEp->curDev_); + + int numBlocks = 0; + CUresult exit_code = cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(&numBlocks, cuFunc, blockSize, dynSMemSize, flags); + if (exit_code == CUDA_SUCCESS) { + tool::Logging(myName, "cudaOccupancyMaxActiveBlocksPerMultiprocessor with Flags success, numBlocks = %d\n", numBlocks); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_OCCUPANCY_MAX_ACTIVE_BLOCKS_PER_MULTIPROCESSOR_WITH_FLAGS); + resBuf.Push(numBlocks); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "cudaOccupancyMaxActiveBlocksPerMultiprocessor with Flags failed: %s\n", errorStr); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtStreamCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtStreamCallback.cc new file mode 100644 index 0000000..a49f890 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtStreamCallback.cc @@ -0,0 +1,312 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "CUDARuntimeStreamHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudaStreamCreateHandle) { + tool::Logging(myName, "CUDA_STREAM_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudaStream_t stream; + cudaError_t exit_code = cudaStreamCreate(&stream); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamCreate success, pStream ptr: %p\n", stream); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, stream); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_STREAM_CREATE); + stream = (cudaStream_t)serverEp->GetHandleVirAddr(stream, CUDA_STREAM_CREATE); + resBuf.Push64BitPointer(stream); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamCreate failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + + +DEFINE_SERVER_AM_CALLBACK(cudaStreamDestroyHandle) { + tool::Logging(myName, "CUDA_STREAM_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudaError_t exit_code = cudaStreamDestroy(stream); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamDestroy success, Stream ptr: %p\n", stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamDestroy failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + + +DEFINE_SERVER_AM_CALLBACK(cudaStreamCreateWithFlagsHandle) { + tool::Logging(myName, "CUDA_STREAM_CREATE_WITH_FLAGS\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudaStream_t stream; + unsigned int flags = reqBuf.Pop(); + cudaError_t exit_code = cudaStreamCreateWithFlags(&stream, flags); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamCreateWithFlags success, Stream ptr: %p, flags=%u\n", stream, flags); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, stream); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_STREAM_CREATE_WITH_FLAGS); + stream = (cudaStream_t)serverEp->GetHandleVirAddr(stream, CUDA_STREAM_CREATE_WITH_FLAGS); + resBuf.Push64BitPointer(stream); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamCreateWithFlags failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} +// end + +DEFINE_SERVER_AM_CALLBACK(cudaStreamCreateWithPriorityHandle) { + tool::Logging(myName, "CUDA_STREAM_CREATE_WITH_PRIORITY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudaStream_t stream; + unsigned int flags = reqBuf.Pop(); + int priority = reqBuf.Pop(); + cudaError_t exit_code = cudaStreamCreateWithPriority(&stream, flags, priority); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamCreateWithPriority success, Stream ptr: %p, flags = %u, priority = %d\n", stream, flags, priority); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, stream); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_STREAM_CREATE_WITH_PRIORITY); + stream = (cudaStream_t)serverEp->GetHandleVirAddr(stream, CUDA_STREAM_CREATE_WITH_PRIORITY); + resBuf.Push64BitPointer(stream); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamCreateWithPriority failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaStreamWaitEventHandle) { + tool::Logging(myName, "CUDA_STREAM_WAIT_EVENT\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + uint64_t eventVirAddr= reqBuf.Pop(); + cudaEvent_t event = (cudaEvent_t)serverEp->GetHandle(eventVirAddr); + uint flags = reqBuf.Pop(); + if (event == NULL) { + tool::Logging(LOG_ERROR, myName, "cudaStreamWaitEvent warning: eventPtr is NULL(virAddr=%p), streamPtr=%p\n", eventVirAddr, stream); + return UCS_OK; + } + + cudaError_t exit_code = cudaStreamWaitEvent(stream, event, flags); + if (exit_code == cudaSuccess) { + bool isCommEvent = false; +#ifdef GV_BACKUP + Handle_t* handle = serverEp->GetHandleInfo(eventVirAddr); + isCommEvent = (serverEp->commStream_) && (serverEp->commStream_ == (cudaStream_t)handle->stream); + if (isCommEvent) { + boost::unique_lock lock(serverEp->eventWatchedSync_.mutex); + serverEp->watchedEventsList_.push_back(event); + } +#endif // GV_BACKUP + tool::Logging(myName, "cudaStreamWaitEvent success, eventPtr=%p(isCommEvent=%d), streamPtr=%p, flags = %u\n", event, isCommEvent, stream, flags); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamWaitEvent failed: %s, eventPtr=%p(virAddr=%p), streamPtr=%p\n", cudaGetErrorName(exit_code), event, eventVirAddr, stream); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaStreamSynchronizeHandle) { + tool::Logging(myName, "CUDA_STREAM_SYNCHRONIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + +#ifdef GV_BACKUP + // debug: + // if (serverEp->ckptCnt >= 2) { + // // if (serverEp->curIter_ >= BACKUP_PERIOD + 1) { + // // if (serverEp->curIter_ >= BACKUP_PERIOD + BACKUP_PERIOD - 5) { + // tool::Logging(LOG_INFO, myName, "Ready to close the server endpoint object.\n"); + // // { + // // boost::unique_lock lock(serverEp->backupSync_.mutex); + // // serverEp->backupSync_.cv.wait(lock, [serverEp] { return !serverEp->bufferReady_; }); + // // } + // serverEp->stateBackup_.Wait(); + + // for (const auto stream : serverEp->streamList_) { + // cudaStreamSynchronize(stream); + // } + + // /* + // size_t totalTensorsSize = 0; + // for (const auto& tensor : serverEp->trainTensors_) { + // totalTensorsSize += tensor.size; + // } + // totalTensorsSize = totalTensorsSize * BACKUP_PERIOD; + // std::string fileName = serverEp->backupFilePath_ + ".tmp"; + // std::ofstream ofs(fileName.c_str(), std::ios::binary | std::ios::out | std::ios::trunc); + // if (!ofs) { + // throw std::runtime_error("Opening file failed: " + fileName); + // } + // ofs.seekp(totalTensorsSize - 1); + // ofs.write("", 1); + // ofs.close(); + // boost::interprocess::file_mapping fileMapping(fileName.c_str(), boost::interprocess::read_write); + // boost::interprocess::mapped_region mappingRegion(fileMapping, boost::interprocess::read_write, 0, totalTensorsSize); + // auto start = std::chrono::high_resolution_clock::now(); + // std::memcpy(mappingRegion.get_address(), serverEp->backupMemcpyBuffer_, totalTensorsSize); + // auto end = std::chrono::high_resolution_clock::now(); + // tool::Logging(LOG_INFO, myName, "training data is written to file(%s) with size %zu bytes in %f seconds\n", fileName.c_str(), totalTensorsSize, std::chrono::duration(end - start).count()); + // */ + + // // exit(EXIT_FAILURE); + + // serverEp->connStatus_.isClosed = true; + // for (int i = 0; i < 3; i++) serverEp->shmQueSizes_[i]->SaveState(); + // return UCS_ERR_CONNECTION_RESET; + // } +#ifdef GV_MSGHANDLER + // if (serverEp->MSGStopFlag_) { + if (serverEp->MSGStopFlag_ && !serverEp->msgWaitFlag_){ + tool::Logging(LOG_INFO, myName, "Ready to close the server endpoint object.\n"); + serverEp->msgResetCnt = serverEp->ckptCnt; + serverEp->msgWaitFlag_ = true; + } + else if (serverEp->MSGStopFlag_ && serverEp->msgWaitFlag_ && (serverEp->ckptCnt >= serverEp->msgResetCnt + 1)) { + // if (serverEp->curIter_ >= BACKUP_PERIOD + 1) { + // if (serverEp->curIter_ >= BACKUP_PERIOD + BACKUP_PERIOD - 5) { + tool::Logging(LOG_INFO, myName, "Ready to close the server endpoint object.\n"); + // { + // boost::unique_lock lock(serverEp->backupSync_.mutex); + // serverEp->backupSync_.cv.wait(lock, [serverEp] { return !serverEp->bufferReady_; }); + // } + serverEp->stateBackup_.Wait(); + + for (const auto stream : serverEp->streamList_) { + cudaStreamSynchronize(stream); + } + + /* + size_t totalTensorsSize = 0; + for (const auto& tensor : serverEp->trainTensors_) { + totalTensorsSize += tensor.size; + } + totalTensorsSize = totalTensorsSize * BACKUP_PERIOD; + std::string fileName = serverEp->backupFilePath_ + ".tmp"; + std::ofstream ofs(fileName.c_str(), std::ios::binary | std::ios::out | std::ios::trunc); + if (!ofs) { + throw std::runtime_error("Opening file failed: " + fileName); + } + ofs.seekp(totalTensorsSize - 1); + ofs.write("", 1); + ofs.close(); + boost::interprocess::file_mapping fileMapping(fileName.c_str(), boost::interprocess::read_write); + boost::interprocess::mapped_region mappingRegion(fileMapping, boost::interprocess::read_write, 0, totalTensorsSize); + auto start = std::chrono::high_resolution_clock::now(); + std::memcpy(mappingRegion.get_address(), serverEp->backupMemcpyBuffer_, totalTensorsSize); + auto end = std::chrono::high_resolution_clock::now(); + tool::Logging(LOG_INFO, myName, "training data is written to file(%s) with size %zu bytes in %f seconds\n", fileName.c_str(), totalTensorsSize, std::chrono::duration(end - start).count()); + */ + + // exit(EXIT_FAILURE); + + serverEp->connStatus_.isClosed = true; + for (int i = 0; i < 3; i++) serverEp->shmQueSizes_[i]->SaveState(); + return UCS_ERR_CONNECTION_RESET; + } + + #endif // GV_MSGHANDLER +#endif // GV_BACKUP + + cudaError_t exit_code = cudaStreamSynchronize(stream); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_STREAM_SYNCHRONIZE); + resBuf.Push(exit_code); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamSynchronize success, pStream=%p\n",stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamSynchronize failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaStreamIsCapturingHandle) { + tool::Logging(myName, "CUDA_STREAM_IS_CAPTURING\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + enum cudaStreamCaptureStatus pCaptureStatus; + cudaError_t exit_code = cudaStreamIsCapturing(stream, &pCaptureStatus); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamIsCapturing success, pStream=%p, pCaptureStatus=%d\n",stream, pCaptureStatus); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_STREAM_IS_CAPTURING); + resBuf.Push(pCaptureStatus); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamIsCapturing failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} +// end + + +DEFINE_SERVER_AM_CALLBACK(cudaStreamGetCaptureInfoHandle) { + tool::Logging(myName, "CUDA_STREAM_GET_CAPTURE_INFO\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + // debug: 这里pop出来的是pStream, 而非stream + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + enum cudaStreamCaptureStatus pCaptureStatus; + unsigned long long pId; + cudaError_t exit_code = cudaStreamGetCaptureInfo(stream, &pCaptureStatus, &pId); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamGetCaptureInfo success, streamPtr=%p, pCaptureStatus=%d, pId=%llu\n",stream, pCaptureStatus, pId); + // send pCaptureStatus and pId of the stream back + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_STREAM_GET_CAPTURE_INFO); + resBuf.Push(pCaptureStatus); + resBuf.Push(pId); //todo: handle mapping + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamGetCaptureInfo failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + + + diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/ncclCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/ncclCallback.cc new file mode 100644 index 0000000..1f250ec --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/ncclCallback.cc @@ -0,0 +1,689 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "ncclHandle"; + +DEFINE_SERVER_AM_CALLBACK(ncclMemAllocHandle) { + tool::Logging(myName, "NCCL_MEM_ALLOC\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + void* devPtr = NULL; + size_t cuda_size = reqBuf.Pop(); + ncclResult_t exit_code = ncclMemAlloc(&devPtr, cuda_size); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclMemAlloc success, devPtr = %p with size = %zu\n", devPtr, cuda_size); + uint64_t virtAddr = (uint64_t)devPtr; + +#ifdef GV_MEMORY + virtAddr = serverEp->_cuInfoMap.blockManager->AddBlock((uint64_t)devPtr, cuda_size, true); +#else + size_t blockNum = ( + serverEp->_cuInfoMap.blockInfoList.emplace_back(Block_t{.devPtr = (uint64_t)devPtr, .size = cuda_size, .valid = true}), + serverEp->_cuInfoMap.blockInfoList.size() - 1 + ); +#endif // GV_MEMORY + +#ifdef GV_MEMORY_PTX + if (blockNum >= BLOCKS_MAX_NUM) { + tool::Logging(LOG_ERROR, myName, "ncclMemAlloc failed: current blockNum(%zu) >= BLOCKS_MAX_NUM(%zu)\n", blockNum, BLOCKS_MAX_NUM); + return UCS_ERR_IO_ERROR; + } + virtAddr = blockNum << BLOCK_SHIFT_BIT; +#endif // GV_MEMORY_PTX + + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_MEM_ALLOC); + // resBuf.Push64BitPointer(devPtr); + resBuf.Push(virtAddr); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclMemAlloc failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclMemFreeHandle) { + tool::Logging(myName, "NCCL_MEM_FREE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virtAddr = reqBuf.Pop(); + void* devPtr = serverEp->GetDevPtr(virtAddr); + ncclResult_t exit_code = ncclMemFree(devPtr); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclMemFree success, devPtr: %p\n", devPtr); +#ifdef GV_MEMORY_PTX + size_t blockIdx = serverEp->GetBlockIdx(virtAddr); + serverEp->_cuInfoMap.blockInfoList[blockIdx].valid = false; +#endif // GV_MEMORY + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclMemFree failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclGroupStartHandle) { + tool::Logging(myName, "NCCL_GROUP_START\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclResult_t exit_code = ncclGroupStart(); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclGroupStart success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclGroupStart failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclGroupEndHandle) { + tool::Logging(myName, "NCCL_GROUP_END\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclResult_t exit_code = ncclGroupEnd(); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclGroupEnd success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclGroupEnd failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommInitRankHandle) { + tool::Logging(myName, "NCCL_COMM_INIT_RANK\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + int nranks = reqBuf.Pop(); + ncclUniqueId commId = reqBuf.Pop(); + int rank = reqBuf.Pop(); + ncclComm_t comm = NULL; + ncclResult_t exit_code = ncclCommInitRank(&comm, nranks, commId, rank); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommInitRank success, comm: %p, nranks: %d, rank: %d\n", comm, nranks, rank); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, comm); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_INIT_RANK); + comm = (ncclComm_t)serverEp->GetHandleVirAddr(comm, NCCL_COMM_INIT_RANK); + resBuf.Push64BitPointer(comm); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + serverEp->curComm = comm; + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommInitRank failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommInitAllHandle) { + tool::Logging(myName, "NCCL_COMM_INIT_ALL\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t* virAddr = reqBuf.AssignAddr(); + int ndev = reqBuf.Pop(); + const int* devlist = reqBuf.AssignAddr(); + ncclComm_t comms[ndev]; // todo: may be invalid + ncclResult_t exit_code = ncclCommInitAll(comms, ndev, devlist); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommInitAll success, ndev: %d\n", ndev); + if (virAddr[0] != 0) { + for (int i = 0; i < ndev; i++) { + serverEp->SetHandleVirAddr(virAddr[i], comms[i]); + } + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_INIT_ALL); + for (int i = 0; i < ndev; i++) { + comms[i] = (ncclComm_t)serverEp->GetHandleVirAddr(comms[i], NCCL_COMM_INIT_ALL); + + } + resBuf.Push((uint64_t*)comms, ndev); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommInitAll failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommInitRankConfigHandle) { + tool::Logging(myName, "NCCL_COMM_INIT_RANK_CONFIG\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + int nranks = reqBuf.Pop(); + ncclUniqueId commId = reqBuf.Pop(); + int rank = reqBuf.Pop(); + ncclConfig_t* config = reqBuf.AssignAddr(); + char* netName = reqBuf.AssignAddr(); + config->netName = netName; + ncclComm_t comm = NULL; + ncclResult_t exit_code = ncclCommInitRankConfig(&comm, nranks, commId, rank, config); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommInitRankConfig success, comm: %p, nranks: %d, rank: %d\n", comm, nranks, rank); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, comm); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_INIT_RANK_CONFIG); + comm = (ncclComm_t)serverEp->GetHandleVirAddr(comm, NCCL_COMM_INIT_RANK_CONFIG); + resBuf.Push64BitPointer(comm); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + serverEp->curComm = comm; + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommInitRankConfig failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommDestroyHandle) { + tool::Logging(myName, "NCCL_COMM_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop(), true); + ncclResult_t exit_code = ncclCommDestroy(comm); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommDestroy success, comm: %p\n", comm); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommDestroy failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommSplitHandle) { + tool::Logging(myName, "NCCL_COMM_SPLIT\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + int color = reqBuf.Pop(); + int key = reqBuf.Pop(); + uint64_t virAddr = reqBuf.Pop(); + ncclConfig_t* config = reqBuf.AssignAddr(); + char* netName = reqBuf.AssignAddr(); + config->netName = netName; + ncclComm_t newcomm = NULL; + ncclResult_t exit_code = ncclCommSplit(comm, color, key, &newcomm, config); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommSplit success, oldcomm: %p, newcomm: %p\n", comm, newcomm); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, newcomm); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_SPLIT); + newcomm = (ncclComm_t)serverEp->GetHandleVirAddr(newcomm, NCCL_COMM_SPLIT); + resBuf.Push64BitPointer(newcomm); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommSplit failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommFinalizeHandle) { + tool::Logging(myName, "NCCL_COMM_FINALIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + ncclResult_t exit_code = ncclCommFinalize(comm); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommFinalize success, comm: %p\n", comm); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommFinalize failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommGetAsyncErrorHandle) { + tool::Logging(myName, "NCCL_COMM_GET_ASYNC_ERROR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + ncclResult_t asyncError = ncclSuccess; + ncclResult_t exit_code = ncclCommGetAsyncError(comm, &asyncError); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommGetAsyncError success, asyncError: %d\n", asyncError); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_GET_ASYNC_ERROR); + resBuf.Push(asyncError); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommGetAsyncError failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommCountHandle) { + tool::Logging(myName, "NCCL_COMM_COUNT\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + int count = 0; + ncclResult_t exit_code = ncclCommCount(comm, &count); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommCount success, count: %d\n", count); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_COUNT); + resBuf.Push(count); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommCount failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommUserRankHandle) { + tool::Logging(myName, "NCCL_COMM_USER_RANK\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + int rank = 0; + ncclResult_t exit_code = ncclCommUserRank(comm, &rank); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommUserRank success, rank: %d\n", rank); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_USER_RANK); + resBuf.Push(rank); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommUserRank failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommCuDeviceHandle) { + tool::Logging(myName, "NCCL_COMM_CU_DEVICE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + int device = 0; + ncclResult_t exit_code = ncclCommCuDevice(comm, &device); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommCuDevice success, device: %d\n", device); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_CU_DEVICE); + resBuf.Push(device); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommCuDevice failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommAbortHandle) { + tool::Logging(myName, "NCCL_COMM_ABORT\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + ncclResult_t exit_code = ncclCommAbort(comm); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommAbort success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommAbort failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommRegisterHandle) { + tool::Logging(myName, "NCCL_COMM_REGISTER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const ncclComm_t comm = (const ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + void* buff = (void*)reqBuf.Pop(); + size_t size = reqBuf.Pop(); + uint64_t virAddr = reqBuf.Pop(); + void* handle = NULL; + buff = serverEp->GetDevPtr((uint64_t)buff); + ncclResult_t exit_code = ncclCommRegister(comm, buff, size, &handle); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommRegister success, buff: %p, handle: %p\n", buff, handle); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, handle); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_REGISTER); + handle = serverEp->GetHandleVirAddr(handle, NCCL_COMM_REGISTER); + resBuf.Push64BitPointer(handle); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommRegister failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommDeregisterHandle) { + tool::Logging(myName, "NCCL_COMM_DEREGISTER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const ncclComm_t comm = (const ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + void* handle = (void*)serverEp->GetHandle(reqBuf.Pop()); + ncclResult_t exit_code = ncclCommDeregister(comm, handle); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommDeregister success, handle: %p\n", handle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommDeregister failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclGetUniqueIdHandle) { + tool::Logging(myName, "NCCL_GET_UNIQUE_ID\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclUniqueId uniqueId; + ncclResult_t exit_code = ncclGetUniqueId(&uniqueId); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclGetUniqueId success\n"); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_GET_UNIQUE_ID); + resBuf.Push(uniqueId); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclGetUniqueId failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclGetVersionHandle) { + tool::Logging(myName, "NCCL_GET_VERSION\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int version; + ncclResult_t exit_code = ncclGetVersion(&version); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclGetVersion success, version: %d\n", version); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_GET_VERSION); + resBuf.Push(version); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclGetVersion failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } + +} + +DEFINE_SERVER_AM_CALLBACK(ncclAllReduceHandle) { + tool::Logging(myName, "NCCL_ALL_REDUCE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const void* sendbuff = (const void*) reqBuf.Pop(); + void* recvbuff = (void*) reqBuf.Pop(); + size_t count = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + ncclRedOp_t op = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + sendbuff = serverEp->GetDevPtr((uint64_t)sendbuff); + recvbuff = serverEp->GetDevPtr((uint64_t)recvbuff); + + // // debug: + // if (serverEp->ckptCnt >= 2) { + // ncclCommAbort(comm); + // ncclCommDestroy(comm); + // { + // boost::unique_lock lock(serverEp->backupSync_.mutex); + // serverEp->backupSync_.cv.wait(lock, [serverEp] { return !serverEp->bufferReady_; }); + // } + // sleep(3); // wait for the client to finish shrinking of api list + // exit(EXIT_FAILURE); + // // serverEp->connStatus_.isClosed = true; + // // for (int i = 0; i < 3; i++) serverEp->shmQueSizes_[i]->SaveState(); + // // return UCS_ERR_CONNECTION_RESET; + // } + + ncclResult_t exit_code = ncclAllReduce(sendbuff, recvbuff, count, datatype, op, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclAllReduce success, sendbuff: %p, recvbuff: %p, count: %d, stream: %p\n", sendbuff, recvbuff, count, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclAllReduce failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclReduceHandle) { + tool::Logging(myName, "NCCL_REDUCE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const void* sendbuff = (const void*) reqBuf.Pop(); + void* recvbuff = (void*) reqBuf.Pop(); + size_t count = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + ncclRedOp_t op = reqBuf.Pop(); + int root = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + sendbuff = serverEp->GetDevPtr((uint64_t)sendbuff); + recvbuff = serverEp->GetDevPtr((uint64_t)recvbuff); + ncclResult_t exit_code = ncclReduce(sendbuff, recvbuff, count, datatype, op, root, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclReduce success, sendbuff: %p, recvbuff: %p, count: %d, stream: %p\n", sendbuff, recvbuff, count, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclReduce failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclReduceScatterHandle) { + tool::Logging(myName, "NCCL_REDUCE_SCATTER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const void* sendbuff = (const void*) reqBuf.Pop(); + void* recvbuff = (void*) reqBuf.Pop(); + size_t recvcount = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + ncclRedOp_t op = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + sendbuff = serverEp->GetDevPtr((uint64_t)sendbuff); + recvbuff = serverEp->GetDevPtr((uint64_t)recvbuff); + ncclResult_t exit_code = ncclReduceScatter(sendbuff, recvbuff, recvcount, datatype, op, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclReduceScatter success, sendbuff: %p, recvbuff: %p, recvcount: %d, stream: %p\n", sendbuff, recvbuff, recvcount, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclReduceScatter failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclAllGatherHandle) { + tool::Logging(myName, "NCCL_ALL_GATHER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const void* sendbuff = (const void*) reqBuf.Pop(); + void* recvbuff = (void*) reqBuf.Pop(); + size_t sendcount = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + sendbuff = serverEp->GetDevPtr((uint64_t)sendbuff); + recvbuff = serverEp->GetDevPtr((uint64_t)recvbuff); + ncclResult_t exit_code = ncclAllGather(sendbuff, recvbuff, sendcount, datatype, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclAllGather success, sendbuff: %p, recvbuff: %p, sendcount: %d, stream: %p\n", sendbuff, recvbuff, sendcount, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclAllGather failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclBroadcastHandle) { + tool::Logging(myName, "NCCL_BROADCAST\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const void* sendbuff = (const void*) reqBuf.Pop(); + void* recvbuff = (void*) reqBuf.Pop(); + size_t count = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + int root = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + sendbuff = serverEp->GetDevPtr((uint64_t)sendbuff); + recvbuff = serverEp->GetDevPtr((uint64_t)recvbuff); + ncclResult_t exit_code = ncclBroadcast(sendbuff, recvbuff, count, datatype, root, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclBroadcast success, sendbuff: %p, recvbuff: %p, count: %d, stream: %p\n", sendbuff, recvbuff, count, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclBroadcast failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclSendHandle) { + tool::Logging(myName, "NCCL_SEND\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const void* sendbuff = (const void*) reqBuf.Pop(); + size_t count = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + int peer = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + sendbuff = serverEp->GetDevPtr((uint64_t)sendbuff); + ncclResult_t exit_code = ncclSend(sendbuff, count, datatype, peer, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclSend success, sendbuff: %p, count: %d, stream: %p\n", sendbuff, count, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclSend failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclRecvHandle) { + tool::Logging(myName, "NCCL_RECV\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + void* recvbuff = (void*) reqBuf.Pop(); + size_t count = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + int peer = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + recvbuff = serverEp->GetDevPtr((uint64_t)recvbuff); + ncclResult_t exit_code = ncclRecv(recvbuff, count, datatype, peer, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclRecv success, recvbuff: %p, count: %d, stream: %p\n", recvbuff, count, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclRecv failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclRedOpCreatePreMulSumHandle) { + tool::Logging(myName, "NCCL_RED_OP_CREATE_PRE_MUL_SUM\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclRedOp_t virOp = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + ncclScalarResidence_t residence = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + void* scalar = NULL; + ncclRedOp_t op; + if (residence == ncclScalarHostImmediate) { + scalar = reqBuf.AssignAddr(); + } else { + scalar = (void*)reqBuf.Pop(); + scalar = serverEp->GetDevPtr((uint64_t)scalar); + } + ncclResult_t exit_code = ncclRedOpCreatePreMulSum(&op, scalar, datatype, residence, comm); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclRedOpCreatePreMulSum success, scalar: %p, op: %d\n", scalar, op); + if (virOp != 0) { + serverEp->SetNcclRedOpVirAddr(virOp, op); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_RED_OP_CREATE_PRE_MUL_SUM); + op = serverEp->GetNcclRedOpVirAddr(op); + resBuf.Push(op); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclRedOpCreatePreMulSum failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclRedOpDestroyHandle) { + tool::Logging(myName, "NCCL_RED_OP_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclRedOp_t op = serverEp->GetNcclRedOp(reqBuf.Pop(), true); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + ncclResult_t exit_code = ncclRedOpDestroy(op, comm); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclRedOpDestroy success, op: %d\n", op); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclRedOpDestroy failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/serverApp.cc b/GPU-Virtual-Service/gpu-remoting/src/server/serverApp.cc new file mode 100644 index 0000000..d8b7539 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/serverApp.cc @@ -0,0 +1,103 @@ +#include +#include "../../include/configure.h" +#include "../../include/ucpConnection.h" +#include "../../include/serverEndpoint.h" +#include "../../include/conqueue/readerwriterqueue.h" +using namespace std; + +Configure config_("config.json"); +const char* myName = "ServerApp"; +UCPConnection* connectionObj; +vector thList; +// moodycamel::BlockingReaderWriterQueue q(CONN_RESERVED_NUM); +volatile bool isClosed = false; + +void CTRLC(int s) { + tool::Logging(LOG_INFO, myName, "terminate the server with ctrl+c interrupt\n"); + __sync_add_and_fetch_8(&isClosed, 1); +} + +void cleanup() { + // for (auto it : thList) { + // it->join(); + // delete it; + // } + // for (auto it : serverEpList) { + // delete it; + // } + // tool::Logging(LOG_INFO, myName, "clear all server endpoint object.\n"); + + delete connectionObj; + tool::Logging(LOG_INFO, myName, "close the network connection.\n"); +} + +int main(int argc, char* argv[]) { + + struct sigaction sigIntHandler; + sigIntHandler.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &sigIntHandler, 0); // avoid the server crash when writing to a closed connection + sigIntHandler.sa_handler = CTRLC; + sigaction(SIGINT, &sigIntHandler, 0); + + // boost::thread* thTmp; + // boost::thread_attributes attrs; + // attrs.set_stack_size(THREAD_STACK_SIZE); + + int devCnt = 0; + cudaError_t exit_code = cudaGetDeviceCount(&devCnt); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaGetDeviceCount failed: %s\n", cudaGetErrorName(exit_code)); + exit(EXIT_FAILURE); + } + + + connectionObj = new UCPConnection(false); + connectionObj->Listen(config_.GetServerIp(), config_.GetServerPort(), ServerEndpoint::CreateServerEp); + connectionObj->WaitConnection(&isClosed); + + cleanup(); + + // while(true) { + // ucp_conn_request_h conn_request = NULL; + // uint64_t client_id = 0; + // char* clientIP = NULL; + // char* clientPort = NULL; + // connectionObj->WaitConnection(&conn_request, &client_id, &clientIP, &clientPort); + + // // ucp_worker_h dataWorker; + // // connectionObj->InitWorker(&dataWorker); + // // both dataWorker and conn_request are pointers to the struct + // serverEndPointObj = new ServerEndpoint(connectionObj->CreateWorker(), conn_request, client_id, clientIP, clientPort); + + // thTmp = new boost::thread(attrs, boost::bind(&ServerEndpoint::Run, serverEndPointObj)); + // thList.emplace_back(thTmp); + // //serverEpList.push_back(serverEndPointObj); + // } + + // boost::thread CreateServeEp([&]() { + // UCPConnection_t conn; + // while (!isClosed) { + // if (q.wait_dequeue_timed(conn, std::chrono::seconds(2))) { // wakeup for every 2 seconds + // serverEndPointObj = new ServerEndpoint(connectionObj->CreateWorker(), conn); + // thTmp = new boost::thread(attrs, boost::bind(&ServerEndpoint::NewRun, serverEndPointObj)); + // thList.emplace_back(thTmp); + // } + // } + // }); + + // boost::thread WaitConn([&]() { + // while (!isClosed) { + // UCPConnection_t conn; + // connectionObj->WaitConnection(&conn, &isClosed); + // if (isClosed) { + // break; + // } + // q.enqueue(conn); + // } + // }); + + // WaitConn.join(); + // CreateServeEp.join(); + + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/serverBackup.cc b/GPU-Virtual-Service/gpu-remoting/src/server/serverBackup.cc new file mode 100644 index 0000000..278eb97 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/serverBackup.cc @@ -0,0 +1,657 @@ +#include "../../include/serverEndpoint.h" + +void ServerEndpoint::Backup2Memory() { + const char* myName = "Backup"; + tool::Logging(LOG_DEBUG, myName, "ready to backup the memory blocks and the handle mappings.\n"); + + auto start_total = std::chrono::high_resolution_clock::now(); + + // /* Wait for the sub thread to signal that the buffer is ready to be overwritten */ + // { + // boost::unique_lock lock(backupSync_.mutex); + // backupSync_.cv.wait(lock, [this] { return !bufferReady_; }); + // } + + stateBackup_.Wait(); + + /* prepare the backup buffer and prepare */ + auto start_prepare = std::chrono::high_resolution_clock::now(); + _cuInfoMap.handleManager->Shrink(); // remove the invalid handles + size_t totalDataBytes = 0, memcpyDataBytes = 0; + // const std::vector& handles = _cuInfoMap.handleManager->GetHandleInfoList(); + const std::vector>& handles = _cuInfoMap.handleManager->GetValidHandles(); + const std::vector& blocks = _cuInfoMap.blockManager->GetBlocks(); + + std::vector sortedTensors = trainTensors_; + std::sort(sortedTensors.begin(), sortedTensors.end(), + [](const TensorInfo_t& a, const TensorInfo_t& b) { + return a.devPtr < b.devPtr; + }); + std::vector> block2tensors(blocks.size()); + for (size_t i = 0; i < sortedTensors.size(); i++) { + uint64_t realAddr = 0; + int blockIdx = _cuInfoMap.blockManager->FindByVirAddr((uint64_t)sortedTensors[i].devPtr, realAddr); + if (blockIdx != -1) { + const Block_t& tmpBlock = blocks[blockIdx]; + sortedTensors[i].devPtr = (void*)realAddr; + if (tmpBlock.essential && realAddr + sortedTensors[i].size <= tmpBlock.devPtr + tmpBlock.size) { + block2tensors[blockIdx].push_back(&sortedTensors[i]); + tool::Logging(LOG_DEBUG, myName, "block[%d](addr=%p, size=%zu) has tensor[%zu](%p) with size %zu\n", blockIdx, (void*)tmpBlock.devPtr, tmpBlock.size, i, sortedTensors[i].devPtr, sortedTensors[i].size); + memcpyDataBytes += sortedTensors[i].size; + } + else if (tmpBlock.essential == false) { + tool::Logging(LOG_DEBUG, myName, "block[%d](%p) is not essential\n", blockIdx, (void*)tmpBlock.devPtr); + } + else { + tool::Logging(LOG_DEBUG, myName, "tensor[%zu](%p) is not in block[%d](%p)\n", i, sortedTensors[i].devPtr, blockIdx, (void*)tmpBlock.devPtr); + } + } + } + // size_t tmpOffset = 0; + for (size_t blockIdx = 0; blockIdx < blocks.size(); blockIdx++) { + if (blocks[blockIdx].valid == false || blocks[blockIdx].essential == false) { + continue; + } + // tool::Logging(LOG_INFO, myName, "offset=%zu, prepare to backup block[%zu]: devPtr=%p, size=%zu\n", tmpOffset, blockIdx, (void*)blocks[blockIdx].devPtr, blocks[blockIdx].size); + totalDataBytes += blocks[blockIdx].size; + // tmpOffset += blocks[blockIdx].size; + } + + size_t curBackupSize = sizeof(lastCopyLen_) + sizeof(lastCopyType_) + sizeof(size_t) * 3 + sizeof(handles[0]) * handles.size() + + sizeof(Block_t) * blocks.size() + sizeof(TensorInfo_t) * trainTensors_.size() + + (totalDataBytes - memcpyDataBytes); + // + totalDataBytes; + tool::Logging(LOG_DEBUG, myName, "prepare the backup buffer with %zu bytes (dataBytes: %zu, memcpyBytes: %zu)\n", curBackupSize, totalDataBytes, memcpyDataBytes); + if (serverState_.hostPtr == NULL || serverState_.size < curBackupSize) { + bufferResized_ = true; + serverState_.size = ALIGN_UP(curBackupSize); // align to 4KB + if (serverState_.hostPtr != NULL) { + // free(serverState_.hostPtr); + cudaFreeHost(serverState_.hostPtr); + } + cudaError_t exit_code = cudaMallocHost(&serverState_.hostPtr, serverState_.size); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMallocHost failed: %s\n", cudaGetErrorString(exit_code)); + exit(EXIT_FAILURE); + } + } + if (streamList_.size() == 0) { + streamList_.resize(BACKUP_STREAM_NUM, NULL); + for (int i = 0; i < streamList_.size(); i++) { + cudaStreamCreateWithFlags(&streamList_[i], cudaStreamNonBlocking); + } + } + auto end_prepare = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration_prepare = end_prepare - start_prepare; + tool::Logging(LOG_INFO, myName, "backup buffer is prepared with %zu bytes (isResided: %d) in %f seconds\n", serverState_.size, bufferResized_, duration_prepare.count()); + + + /* backup the mappings for the memory blocks and the handles */ + auto start_backup = std::chrono::high_resolution_clock::now(); + size_t bufferOffset = 0; + size_t handleNum = handles.size(); + size_t blockNum = blocks.size(); + size_t memcpyCnt = trainTensors_.size(); + memcpy(serverState_.hostPtr + bufferOffset, &lastCopyLen_, sizeof(lastCopyLen_)); + bufferOffset += sizeof(lastCopyLen_); + memcpy(serverState_.hostPtr + bufferOffset, &lastCopyType_, sizeof(lastCopyType_)); + bufferOffset += sizeof(lastCopyType_); + memcpy(serverState_.hostPtr + bufferOffset, &handleNum, sizeof(size_t)); + bufferOffset += sizeof(size_t); + memcpy(serverState_.hostPtr + bufferOffset, &blockNum, sizeof(size_t)); + bufferOffset += sizeof(size_t); + memcpy(serverState_.hostPtr + bufferOffset, &memcpyCnt, sizeof(size_t)); + bufferOffset += sizeof(size_t); + // memcpy(serverState_.hostPtr + bufferOffset, handles.data(), sizeof(Handle_t) * handleNum); + // bufferOffset += sizeof(Handle_t) * handleNum; + memcpy(serverState_.hostPtr + bufferOffset, handles.data(), sizeof(handles[0]) * handleNum); + bufferOffset += sizeof(handles[0]) * handleNum; + memcpy(serverState_.hostPtr + bufferOffset, blocks.data(), sizeof(Block_t) * blockNum); + bufferOffset += sizeof(Block_t) * blockNum; + memcpy(serverState_.hostPtr + bufferOffset, trainTensors_.data(), sizeof(TensorInfo_t) * trainTensors_.size()); + bufferOffset += sizeof(TensorInfo_t) * trainTensors_.size(); + auto end_backup = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration_backup = end_backup - start_backup; + tool::Logging(LOG_INFO, myName, "backup %zu (valid) handles and %zu blockInfos in %f seconds\n", handleNum, blockNum, duration_backup.count()); + + /* backup the memory blocks */ + auto start = std::chrono::high_resolution_clock::now(); + size_t essentialBlockNum = 0; + for (size_t blockIdx = 0; blockIdx < blocks.size(); blockIdx++) { + if (blocks[blockIdx].valid == false || blocks[blockIdx].essential == false) { + continue; + } + essentialBlockNum++; + + // printf("\n"); + tool::Logging(LOG_DEBUG, myName, "bufferOffset=%zu, prepare to backup block[%zu]: devPtr=%p, size=%zu\n", bufferOffset, blockIdx, (void*)blocks[blockIdx].devPtr, blocks[blockIdx].size); + + const Block_t& block = blocks[blockIdx]; + const auto& tensorsInBlock = block2tensors[blockIdx]; + int streamIdx = blockIdx % streamList_.size(); + uint8_t* blockAddr = (uint8_t*)block.devPtr; + + for (const auto& tensor : tensorsInBlock) { + uint8_t* tensorStart = (uint8_t*)tensor->devPtr; + uint8_t* tensorEnd = tensorStart + tensor->size; + + // copy the gap between the last tensor(or the start of the block) and the current tensor + size_t lastSegmentSize = tensorStart - blockAddr; + if (lastSegmentSize > 0) { + cudaError_t memcpy_exit_code = cudaMemcpyAsync(serverState_.hostPtr + bufferOffset, (void*)blockAddr, lastSegmentSize, cudaMemcpyDeviceToHost, streamList_[streamIdx]); + if (memcpy_exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync failed for the memory segment in block[%zu] (%p : %p) with %zu bytes: %s\n", blockIdx, blockAddr, tensorStart, lastSegmentSize, cudaGetErrorString(memcpy_exit_code)); + exit(EXIT_FAILURE); + } + else { + bufferOffset += lastSegmentSize; + tool::Logging(LOG_DEBUG, myName, "backup the memory segment in block[%zu] (%p : %p) with %zu bytes\n", blockIdx, blockAddr, tensorStart, lastSegmentSize); + } + } + // move the blockAddr to the end of the current tensor + blockAddr = tensorEnd; + } + + // copy the remaining data in the block + size_t remainingSize = (uint8_t*)block.devPtr+block.size - blockAddr; + if (remainingSize > 0) { + cudaError_t memcpy_exit_code = cudaMemcpyAsync(serverState_.hostPtr + bufferOffset, (void*)blockAddr, remainingSize, cudaMemcpyDeviceToHost, streamList_[streamIdx]); + if (memcpy_exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync failed for the remaining memory segment in block[%zu] (%p) with %zu bytes: %s\n", blockIdx, blockAddr, remainingSize, cudaGetErrorString(memcpy_exit_code)); + tool::Logging(LOG_ERROR, myName, "bufferOffset=%zu, essentialBlockNum=%zu\n", bufferOffset, essentialBlockNum); + exit(EXIT_FAILURE); + } + else { + tool::Logging(LOG_DEBUG, myName, "backup the remaining memory segment in block[%zu] (%p) with %zu bytes\n", blockIdx, blockAddr, remainingSize); + bufferOffset += remainingSize; + essentialBlockNum++; + } + } + } + for (int i = 0; i < streamList_.size(); i++) { + cudaError_t sync_code = cudaStreamSynchronize(streamList_[i]); + if (sync_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaStreamSynchronize failed: %s\n", cudaGetErrorString(sync_code)); + exit(EXIT_FAILURE); + } + } + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration = end - start; + tool::Logging(LOG_INFO, myName, "backup device memory success with %zu bytes (%zu essential blocks) in %f seconds\n", bufferOffset, essentialBlockNum, duration.count()); + + const char* backupcsv_path = std::getenv("FLEXGV_BACKUP_CSV_PATH"); + if (backupcsv_path != NULL) { + std::ofstream outFile; + outFile.open(backupcsv_path, std::ios::app); + if (outFile.tellp() == 0) { + outFile << "Bytes,Time" << std::endl; + } + outFile << bufferOffset << "," << duration.count() << std::endl; + outFile.close(); + } + + // /* Signal the sub thread that the buffer is ready to be written */ + // { + // boost::unique_lock lock(backupSync_.mutex); + // bufferReady_ = true; + // backupSync_.cv.notify_one(); + // } + + stateBackup_.Notify(); + + // debug: + // std::vector& tmpBlocks = _cuInfoMap.blockManager->GetBlocks(); + // for (size_t blockIdx = 0; blockIdx < tmpBlocks.size(); blockIdx++) { + // if (tmpBlocks[blockIdx].valid == false) { + // continue; + // } + // tool::Logging(LOG_DEBUG, myName, "blockInfoList[%zu]: devPtr=%p, size=%zu, valid=%d\n", blockIdx, (void*)tmpBlocks[blockIdx].devPtr, tmpBlocks[blockIdx].size, tmpBlocks[blockIdx].valid); + // cudaFree((void*)tmpBlocks[blockIdx].devPtr); + // tmpBlocks[blockIdx].valid = false; + // tmpBlocks[blockIdx].devPtr = (uint64_t)NULL; + // tmpBlocks[blockIdx].size = 0; + // } + // std::vector& tmpHandles = _cuInfoMap.handleManager->GetHandleInfoList(); + // for (size_t i = 1; i < tmpHandles.size(); i++) { + // if (tmpHandles[i].valid == false) { + // continue; + // } + // tool::Logging(LOG_INFO, myName, "handleInfoList[%zu]: handlePtr=%p, type=%d, valid=%d\n", i, (void*)tmpHandles[i].handlePtr, tmpHandles[i].type, tmpHandles[i].valid); + // } +} + +void ServerEndpoint::Persist2File(const char* fileName, const uint8_t* data, size_t size) { + try { + if (bufferResized_) { + bufferResized_ = false; + /* resize or create the file */ + std::ofstream ofs(fileName, std::ios::binary | std::ios::out | std::ios::trunc); + if (!ofs) { + throw std::runtime_error("Opening file failed: " + std::string(fileName)); + } + ofs.seekp(size - 1); + ofs.write("", 1); + ofs.close(); + tool::Logging(LOG_DEBUG, "Persist2File", "File(%s) is resized to %zu bytes\n", fileName, size); + + /* open the file and map to memory */ + boost::interprocess::file_mapping fileMapping(fileName, boost::interprocess::read_write); + fileMappingRegion_ = std::make_unique(fileMapping, boost::interprocess::read_write, 0, size); + } + + // copy the data to the memory-mapped file + auto start = std::chrono::high_resolution_clock::now(); + std::memcpy(fileMappingRegion_->get_address(), data, size); + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration = end - start; + tool::Logging(LOG_INFO, "Persist2File", "Time taken to copy data to file(%s): %f seconds\n", fileName, duration.count()); + } + catch (const boost::interprocess::interprocess_exception& ex) { + tool::Logging(LOG_ERROR, "Persist2File", "Writing to file failed: %s\n", ex.what()); + } + catch (const std::exception& ex) { + tool::Logging(LOG_ERROR, "Persist2File", "Writing to file failed: general error(%s)\n", ex.what()); + } +} + +void ServerEndpoint::Backup2Storage() { + while (true) { + // /* Wait for main thread to signal that the buffer is ready or the thread should exit */ + // { + // boost::unique_lock lock(backupSync_.mutex); + // backupSync_.cv.wait(lock, [this] { return bufferReady_ || bufferFinished_; }); + // if (bufferFinished_ && !bufferReady_) { + // break; + // } + // } + if (!stateBackup_.Check()) { + break; + } + + /* Write the buffer to storage */ + Persist2File(backupFilePath_.c_str(), serverState_.hostPtr, serverState_.size); + + // /* Signal the main thread that the buffer has been written */ + // { + // boost::unique_lock lock(backupSync_.mutex); + // bufferReady_ = false; + // backupSync_.cv.notify_one(); + // } + + stateBackup_.Notify(); + } +} + +void ServerEndpoint::LoadFromStorage() { + const char* myName = "Load"; + const char* fileName = backupFilePath_.c_str(); + tool::Logging(LOG_DEBUG, myName, "ready to load the memory blocks and the handle mappings.\n"); + if (streamList_.size() == 0) { + streamList_.resize(BACKUP_STREAM_NUM, NULL); + for (int i = 0; i < streamList_.size(); i++) { + cudaStreamCreateWithFlags(&streamList_[i], cudaStreamNonBlocking); + } + } + try { +/* + std::string tmpFileName = backupFilePath_ + ".tmp"; + std::ifstream tmpofs(tmpFileName, std::ios::binary | std::ios::ate); + if (!tmpofs.is_open()) { + throw std::runtime_error(std::string("Unable to open file: ") + tmpFileName); + } + std::streamsize tmpFileSize = tmpofs.tellg(); + if (tmpFileSize < 0) { + throw std::runtime_error("Unable to get file size"); + } + tmpofs.close(); + size_t trainDataSize = static_cast(tmpFileSize); + backupMemcpyBuffer_ = malloc(trainDataSize); + tool::Logging(LOG_DEBUG, myName, "backupMemcpyBuffer_ is allocated with %zu bytes\n", trainDataSize); + boost::interprocess::file_mapping tmpFile(tmpFileName.c_str(), boost::interprocess::read_only); + boost::interprocess::mapped_region tmpRegion(tmpFile, boost::interprocess::read_only, 0, trainDataSize); + std::memcpy(backupMemcpyBuffer_, tmpRegion.get_address(), trainDataSize); + tool::Logging(LOG_INFO, myName, "backupMemcpyBuffer_ is loaded with %zu bytes\n", trainDataSize); +*/ + + /* Get the file size */ + std::ifstream ofs(fileName, std::ios::binary | std::ios::ate); + if (!ofs.is_open()) { + throw std::runtime_error(std::string("Unable to open file: ") + fileName); + } + std::streamsize fileSize = ofs.tellg(); + if (fileSize < 0) { + throw std::runtime_error("Unable to get file size"); + } + ofs.close(); + size_t dataSize = static_cast(fileSize); + + boost::interprocess::file_mapping file(fileName, boost::interprocess::read_only); + boost::interprocess::mapped_region region(file, boost::interprocess::read_only, 0, dataSize); + + size_t bufferOffset = 0; + size_t handleNum = 0, blockNum = 0, memcpyCnt = 0; + memcpy(&lastCopyLen_, (uint8_t*)region.get_address() + bufferOffset, sizeof(lastCopyLen_)); + bufferOffset += sizeof(lastCopyLen_); + memcpy(&lastCopyType_, (uint8_t*)region.get_address() + bufferOffset, sizeof(lastCopyType_)); + bufferOffset += sizeof(lastCopyType_); + memcpy(&handleNum, (uint8_t*)region.get_address() + bufferOffset, sizeof(size_t)); + bufferOffset += sizeof(size_t); + memcpy(&blockNum, (uint8_t*)region.get_address() + bufferOffset, sizeof(size_t)); + bufferOffset += sizeof(size_t); + memcpy(&memcpyCnt, (uint8_t*)region.get_address() + bufferOffset, sizeof(size_t)); + bufferOffset += sizeof(size_t); + tool::Logging(LOG_INFO, myName, "ready to load %zu (valid) handles and %zu blocks\n", handleNum, blockNum); + + // std::vector& tmpHandleList = _cuInfoMap.handleManager->GetHandleInfoList(); + std::vector> tmpHandleList; + tmpHandleList.resize(handleNum); + // std::memcpy(tmpHandleList.data(), (uint8_t*)region.get_address() + bufferOffset, sizeof(Handle_t) * handleNum); + std::memcpy(tmpHandleList.data(), (uint8_t*)region.get_address() + bufferOffset, sizeof(tmpHandleList[0]) * handleNum); + // bufferOffset += sizeof(Handle_t) * handleNum; + bufferOffset += sizeof(tmpHandleList[0]) * handleNum; + _cuInfoMap.handleManager->Reset(tmpHandleList); + tool::Logging(LOG_INFO, myName, "handleInfoList is successfully loaded with %zu handles\n", handleNum); + // _cuInfoMap.handleManager->Indexing(); + // _cuInfoMap.handleManager->Shrink(); + + std::vector& tmpBlockList = _cuInfoMap.blockManager->GetBlocks(); + tmpBlockList.resize(blockNum); + std::memcpy(tmpBlockList.data(), (uint8_t*)region.get_address() + bufferOffset, sizeof(Block_t) * blockNum); + bufferOffset += sizeof(Block_t) * blockNum; + tool::Logging(LOG_INFO, myName, "blockInfoList is successfully loaded with %zu blocks\n", blockNum); + + trainTensors_.resize(memcpyCnt); + std::memcpy(trainTensors_.data(), (uint8_t*)region.get_address() + bufferOffset, sizeof(TensorInfo_t) * memcpyCnt); + bufferOffset += sizeof(TensorInfo_t) * memcpyCnt; + + std::vector sortedTensors = trainTensors_; + std::sort(sortedTensors.begin(), sortedTensors.end(), + [](const TensorInfo_t& a, const TensorInfo_t& b) { + return a.devPtr < b.devPtr; + }); + std::vector> block2tensors(tmpBlockList.size()); + for (size_t i = 0; i < sortedTensors.size(); i++) { + uint64_t tmpAddr = 0; + int blockIdx = _cuInfoMap.blockManager->FindByVirAddr((uint64_t)sortedTensors[i].devPtr, tmpAddr); + if (blockIdx != -1) { + const Block_t& tmpBlock = tmpBlockList[blockIdx]; + if (tmpBlock.essential && (uint8_t*)sortedTensors[i].devPtr + sortedTensors[i].size <= (uint8_t*)tmpBlock.start + tmpBlock.size) { + block2tensors[blockIdx].push_back(&sortedTensors[i]); + } //!: the devPtr of the sortedTensors is virtual address because the real address is not available + } + } + + auto start = std::chrono::high_resolution_clock::now(); + size_t essentialBlockNum = 0; + for (size_t blockIdx = 0; blockIdx < tmpBlockList.size(); blockIdx++) { + if (tmpBlockList[blockIdx].valid == false) { + continue; + } + Block_t* block = &tmpBlockList[blockIdx]; + void* devPtr = NULL; + cudaError_t exit_code = cudaMalloc(&devPtr, block->size); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMalloc failed: %s\n", cudaGetErrorString(exit_code)); + exit(EXIT_FAILURE); + } + else { + block->devPtr = (uint64_t)devPtr; + tool::Logging(LOG_DEBUG, myName, "blockInfoList[%zu]: devPtr=%p (virAddr=%p:%p), size=%zu, valid=%d, essential=%d\n", blockIdx, (void*)block->devPtr, (void*)block->start, (uint8_t*)block->start+block->size, block->size, block->valid, block->essential); + } + + if (tmpBlockList[blockIdx].essential == false) { + continue; + } + essentialBlockNum++; + + const auto& tensorsInBlock = block2tensors[blockIdx]; + int streamIdx = blockIdx % streamList_.size(); + uint8_t* blockAddr = (uint8_t*)block->devPtr; + + for (const auto& tensor: tensorsInBlock) { + uint8_t* tensorStart = (uint8_t*)devPtr + ((uint64_t)tensor->devPtr - block->start); // the real address of the tensor + uint8_t* tensorEnd = tensorStart + tensor->size; + + // copy the gap between the last tensor(or the start of the block) and the current tensor + size_t lastSegmentSize = tensorStart - blockAddr; + if (lastSegmentSize > 0) { + cudaError_t memcpy_exit_code = cudaMemcpyAsync((void*)blockAddr, (uint8_t*)region.get_address() + bufferOffset, lastSegmentSize, cudaMemcpyHostToDevice, streamList_[streamIdx]); + if (memcpy_exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync failed for the memory segment in block[%zu] (%p : %p) with %zu bytes: %s\n", blockIdx, blockAddr, tensorStart, lastSegmentSize, cudaGetErrorString(memcpy_exit_code)); + exit(EXIT_FAILURE); + } + else { + bufferOffset += lastSegmentSize; + tool::Logging(LOG_DEBUG, myName, "load the memory segment in block[%zu] (%p : %p) with %zu bytes\n", blockIdx, blockAddr, tensorStart, lastSegmentSize); + } + } + // move the blockAddr to the end of the current tensor + blockAddr = tensorEnd; + } + // copy the remaining data in the block + size_t remainingSize = (uint8_t*)block->devPtr + block->size - blockAddr; + if (remainingSize > 0) { + cudaError_t memcpy_exit_code = cudaMemcpyAsync((void*)blockAddr, (uint8_t*)region.get_address() + bufferOffset, remainingSize, cudaMemcpyHostToDevice, streamList_[streamIdx]); + if (memcpy_exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync failed for the remaining memory segment in block[%zu] (%p) with %zu bytes: %s\n", blockIdx, blockAddr, remainingSize, cudaGetErrorString(memcpy_exit_code)); + exit(EXIT_FAILURE); + } + else { + tool::Logging(LOG_DEBUG, myName, "load the remaining memory segment in block[%zu] (%p) with %zu bytes\n", blockIdx, blockAddr, remainingSize); + bufferOffset += remainingSize; + } + } + } + + for (int i = 0; i < streamList_.size(); i++) { + cudaError_t sync_code = cudaStreamSynchronize(streamList_[i]); + if (sync_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaStreamSynchronize failed: %s\n", cudaGetErrorString(sync_code)); + exit(EXIT_FAILURE); + } + } + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration = end - start; + tool::Logging(LOG_INFO, myName, "load device memory success with %zu bytes (%zu essential blocks) in %f seconds\n", bufferOffset, essentialBlockNum, duration.count()); + + // debug: + std::vector& tmpBlocks = _cuInfoMap.blockManager->GetBlocks(); + // for (size_t blockIdx = 0; blockIdx < tmpBlocks.size(); blockIdx++) { + // if (tmpBlocks[blockIdx].valid == false) { + // continue; + // } + // tool::Logging(LOG_INFO, myName, "blockInfoList[%zu]: devPtr=%p, size=%zu, valid=%d\n", blockIdx, (void*)tmpBlocks[blockIdx].devPtr, tmpBlocks[blockIdx].size, tmpBlocks[blockIdx].valid); + // } + // std::vector& tmpHandles = _cuInfoMap.handleManager->GetHandleInfoList(); + // for (size_t i = 1; i < tmpHandles.size(); i++) { + // if (tmpHandles[i].valid == false) { + // continue; + // } + // tool::Logging(LOG_INFO, myName, "handleInfoList[%zu]: handlePtr=%p, type=%d, valid=%d\n", i, (void*)tmpHandles[i].handlePtr, tmpHandles[i].type, tmpHandles[i].valid); + // } + } + catch (const boost::interprocess::interprocess_exception& ex) { + tool::Logging(LOG_ERROR, myName, "Reading from file failed: %s\n", ex.what()); + } + catch (const std::exception& ex) { + tool::Logging(LOG_ERROR, myName, "Reading from file failed: general error(%s)\n", ex.what()); + } +} + +void ServerEndpoint::BackupTrainTensors2Storage() { + const char* myName = "BackupTraintensors"; + size_t miniBatchSize = 0; + for (const auto& tensor : trainTensors_) { + miniBatchSize += tensor.size; + } + if (trainTensorsMappingRegion_ == nullptr) { + std::string fileName = backupFilePath_ + ".tmp"; + const char* fileNamecStr = fileName.c_str(); + std::ofstream ofs(fileName, std::ios::binary | std::ios::out | std::ios::trunc); + if (!ofs) { + throw std::runtime_error("Opening file failed: " + fileName); + } + ofs.seekp(miniBatchSize * BACKUP_PERIOD - 1); + ofs.write("", 1); + ofs.close(); + tool::Logging(LOG_DEBUG, myName, "File(%s) is resized to %zu bytes\n", fileNamecStr, miniBatchSize * BACKUP_PERIOD); + + /* open the file and map to memory */ + boost::interprocess::file_mapping fileMapping(fileNamecStr, boost::interprocess::read_write); + trainTensorsMappingRegion_ = std::make_unique(fileMapping, boost::interprocess::read_write, 0, miniBatchSize * BACKUP_PERIOD); + } + + while (true) { + if (!trainTensorBackup_.Check()) { + break; + } + + tool::Logging(LOG_DEBUG, myName, "start to backup the train tensors for iteration#%zu\n", curIter_); + + auto start = std::chrono::high_resolution_clock::now(); + size_t offset = (curIter_ % BACKUP_PERIOD) * miniBatchSize; + for (size_t i = 0; i < trainTensors_.size(); i++) { + int streamIdx = i % streamList_.size(); + // copy the data to the memory-mapped file + cudaError_t exit_code = cudaMemcpyAsync((uint8_t*)trainTensorsMappingRegion_->get_address() + offset, GetDevPtr((uint64_t)trainTensors_[i].devPtr), trainTensors_[i].size, cudaMemcpyDeviceToHost, streamList_[streamIdx]); // backup in the non-blocking stream + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync failed for the memory segment in tensor[%zu] (%p) with %zu bytes: %s\n", i, trainTensors_[i].devPtr, trainTensors_[i].size, cudaGetErrorString(exit_code)); + exit(EXIT_FAILURE); + } + else { + tool::Logging(LOG_DEBUG, myName, "backup the memory segment in tensor[%zu] (%p) with %zu bytes\n", i, trainTensors_[i].devPtr, trainTensors_[i].size); + + cudaError_t sync_code = cudaStreamSynchronize(streamList_[streamIdx]); + if (sync_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaStreamSynchronize failed: %s\n", cudaGetErrorString(sync_code)); + exit(EXIT_FAILURE); + } + } + offset += trainTensors_[i].size; + } + auto end = std::chrono::high_resolution_clock::now(); + tool::Logging(LOG_DEBUG, myName, "backup the train tensors for iteration#%zu in %f seconds\n", curIter_, std::chrono::duration(end - start).count()); + + trainTensorBackup_.Notify(); + } +} + +void ServerEndpoint::LoadFromStorage(size_t iter) { + const char* myName = "LoadTrainTensors"; + size_t miniBatchSize = 0; + for (const auto& tensor : trainTensors_) { + miniBatchSize += tensor.size; + } + if (trainTensorsMappingRegion_ == nullptr) { + std::string fileName = backupFilePath_ + ".tmp"; + std::ifstream ofs(fileName, std::ios::binary | std::ios::ate); + if (!ofs.is_open()) { + throw std::runtime_error(std::string("Unable to open file: ") + fileName); + } + std::streamsize fileSize = ofs.tellg(); + if (fileSize < 0) { + throw std::runtime_error("Unable to get file size"); + } + ofs.close(); + size_t dataSize = static_cast(fileSize); + + boost::interprocess::file_mapping fileMapping(fileName.c_str(), boost::interprocess::read_write); + trainTensorsMappingRegion_ = std::make_unique(fileMapping, boost::interprocess::read_write, 0, dataSize); + } + size_t offset = (iter % BACKUP_PERIOD) * miniBatchSize; + for (const auto& tensor : trainTensors_) { + cudaError_t exit_code = cudaMemcpyAsync(GetDevPtr((uint64_t)tensor.devPtr), (uint8_t*)trainTensorsMappingRegion_->get_address() + offset, tensor.size, cudaMemcpyHostToDevice, defaultStream_); // loading in the default stream because it is in the critical path + offset += tensor.size; + + cudaError_t sync_code = cudaStreamSynchronize(defaultStream_); + if (sync_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaStreamSynchronize failed: %s\n", cudaGetErrorString(sync_code)); + exit(EXIT_FAILURE); + } + } + tool::Logging(LOG_INFO, myName, "load the train tensors for iteration#%zu\n", iter); +} + +void ServerEndpoint::StopBackup() { + // // notify the sub thread to exit + // { + // boost::unique_lock lock(backupSync_.mutex); + // bufferReady_ = false; + // bufferFinished_ = true; + // backupSync_.cv.notify_one(); + // } + // backupStorageThread_->join(); + + cudaFreeHost(serverState_.hostPtr); + cudaFreeHost(backupMemcpyBuffer_); + for (int i = 0; i < streamList_.size(); i++) { + cudaStreamDestroy(streamList_[i]); + } +} + +void ServerEndpoint::CommEventMonitor() { + while (true) { + std::list tmpList; + + boost::unique_lock lock(eventWatchedSync_.mutex); + if (eventWatchedSync_.cv.wait_for(lock, boost::chrono::seconds(COMM_EVENT_TIMEOUT), + [this] { return bufferFinished_;})) { + tool::Logging(LOG_INFO, "CommEventMonitor", "stop watching events\n"); + break; + } + + tmpList = watchedEventsList_; + if (!notCompleteEventsList_.empty()) { + tool::Logging(LOG_INFO, "CommEventMonitor", "watching %zu events (not complete in last round)\n", notCompleteEventsList_.size()); + for (const auto& elem : notCompleteEventsList_) { + Handle_t* eventHandle = _cuInfoMap.handleManager->GetHandleInfoByRealAddr(elem); + if (eventHandle == NULL || eventHandle->valid == false) { + continue; + } + + cudaError_t exit_code = cudaEventQuery(elem); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, "CommEventMonitor", "cudaEventQuery failed for event(%p): %s\n", (void*)elem, cudaGetErrorString(exit_code)); + // ncclCommAbort(curComm); // trigger segmentation fault + std::terminate(); // as stucking in the callback function, so need to use this to exit + + break; + } + } + // Normally, notCompleteEventsList_ should be empty after the above loop + notCompleteEventsList_.clear(); + } + + if (tmpList.empty()) { + continue; + } + tool::Logging(LOG_INFO, "CommEventMonitor", "watching %zu events\n", tmpList.size()); + for (const auto& elem : tmpList) { + watchedEventsList_.remove(elem); + Handle_t* eventHandle = _cuInfoMap.handleManager->GetHandleInfoByRealAddr(elem); + tool::Logging(LOG_DEBUG, "CommEventMonitor", "watching event(%p from %p)\n", (void*)elem, (void*)eventHandle); + + if (eventHandle == NULL || eventHandle->valid == false) { + // cudaEventDestroy(elem); // destroyed by cudaEventDestroyHandle + continue; + } + + cudaError_t exit_code = cudaEventQuery(elem); + if (exit_code == cudaSuccess) { + // cudaEventDestroy(elem); // destroyed by cudaEventDestroyHandle + } + else { + notCompleteEventsList_.push_back(elem); + } + } + } +} + +void ServerEndpoint::StopCommEventMonitor() { + { + boost::unique_lock lock(eventWatchedSync_.mutex); + bufferFinished_ = true; + eventWatchedSync_.cv.notify_one(); + } + eventWatchedThread_->join(); +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/serverEndpoint.cc b/GPU-Virtual-Service/gpu-remoting/src/server/serverEndpoint.cc new file mode 100644 index 0000000..4532855 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/serverEndpoint.cc @@ -0,0 +1,513 @@ +#include "../../include/serverEndpoint.h" + +ServerEndpoint::ServerEndpoint(ucp_worker_h dataWorker, UCPConnection_t conn){ + _dataWorker = dataWorker; + _connectReq = conn.conn_request; + clientID_ = conn.client_id; + clientIP_ = conn.client_ip; + clientPort_ = conn.client_port; + string wholeName = myShortName_ + std::string(" ") + clientIP_ + std::string(":") + clientPort_; + myName_ = strdup(wholeName.c_str()); + + _cuInfoMap.isFirstIter = true; + _cuInfoMap.mapFatBinHandle2CuModule = new robin_hood::unordered_flat_map(); + _cuInfoMap.mapHost2CuFunc = new robin_hood::unordered_flat_map(); + _cuInfoMap.mapDevName2DevPtr = new robin_hood::unordered_flat_map(); + _cuInfoMap.mapHostVar2CuDevPtr = new robin_hood::unordered_flat_map(); + +#ifdef GV_HANDLE + _cuInfoMap.handleManager = new HandleMapper(); +#endif // GV_HANDLE + +#ifdef GV_MEMORY + _cuInfoMap.blockManager = new DeviceBlockMapper(); +#else + _cuInfoMap.blockInfoList.reserve(1000); + _cuInfoMap.blockInfoList.emplace_back(Block_t{.devPtr = (uint64_t)NULL, .size = 0, .valid = true}); // let the blockNum start from 1 +#endif // GV_MEMORY + + // _cuInfoMap.ncclUidList.reserve(5); + // _cuInfoMap.ncclUidList.emplace_back(ncclUniqueId{}); // let the ncclUidNum start from 1 + _cuInfoMap.ncclRedOpList.resize(ncclNumOps + 1, static_cast(0)); + for (int i = 0; i < ncclNumOps; i++) { + _cuInfoMap.ncclRedOpList[i] = static_cast(i); + } + + + /* __cudaRegister will initialize shmQueSizes_, imageQueue and ptxExtractor */ + _cuInfoMap.imageQueue = nullptr; + _cuInfoMap.ptxExtractor = nullptr; + _fatbinList.reserve(240); + + trainTensors_.reserve(5); + boost::thread_attributes attrs; + attrs.set_stack_size(THREAD_STACK_SIZE); +#ifdef GV_BACKUP + stateBackup_.Start(boost::bind(&ServerEndpoint::Backup2Storage, this)); + // backupStorageThread_ = new boost::thread(attrs, boost::bind(&ServerEndpoint::Backup2Storage, this)); + eventWatchedThread_ = new boost::thread(attrs, boost::bind(&ServerEndpoint::CommEventMonitor, this)); +#endif // GV_BACKUP + +#ifdef GV_Scheduler + sche = new Scheduler(clientID_); +#endif + +#ifdef GV_eScheduler + esche = new eScheduler(clientID_); +#endif + +#ifdef GV_MSGHANDLER + msghandler = new MsgHandler(clientID_); +#endif + + /* The client side should have initiated the connection, leading to this ep's creation */ + ucp_ep_params_t ep_params; + ucs_status_t status; + ep_params.field_mask = UCP_EP_PARAM_FIELD_ERR_HANDLER | + UCP_EP_PARAM_FIELD_CONN_REQUEST; + ep_params.conn_request = _connectReq; + ep_params.err_handler.cb = ErrorCallback; + ep_params.err_handler.arg = &connStatus_; + if ((status = ucp_ep_create(_dataWorker, &ep_params, &_clientEp)) != UCS_OK) { + tool::Logging(LOG_ERROR, myName_, "failed to create an endpoint (%s)\n", ucs_status_string(status)); + ucp_worker_destroy(_dataWorker); + free(_connectReq); // todo + } + + tool::Logging(LOG_COMM, myName_, "create the Server endpoint object (connected to client#%llu %s:%s).\n", clientID_, clientIP_, clientPort_); +} + +void ServerEndpoint::NewRun() { + tool::Logging(LOG_INFO, myName_, "start the server endpoint to serve the client#%llu.\n", clientID_); + connStatus_.isClosed = false; + + RegisterHandler(__CUDA_REGISTER, __cudaRegisterHandle, _dataWorker, this); + RegisterHandler(__CUDA_REGISTER_FAT_BINARY, __cudaRegisterFatBinaryHandle, _dataWorker, this); + RegisterHandler(__CUDA_REGISTER_FUNCTION, __cudaRegisterFunctionHandle, _dataWorker, this); + RegisterHandler(__CUDA_REGISTER_VAR, __cudaRegisterVarHandle, _dataWorker, this); + RegisterHandler(CUDA_LAUNCH_KERNEL, cudaLaunchKernelHandle, _dataWorker, this); + RegisterHandler(CUDA_FUNC_GET_ATTRIBUTES, cudaFuncGetAttributesHandle, _dataWorker, this); + RegisterHandler(CUDA_MALLOC, cudaMallocHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMSET, cudaMemsetHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMSET_ASYNC, cudaMemsetAsyncHandle, _dataWorker, this); + RegisterHandler(CUDA_MEM_GET_INFO, cudaMemGetInfoHandle, _dataWorker, this); + RegisterHandler(CUDA_FREE, cudaFreeHandle, _dataWorker, this); + // RegisterHandler(CUDA_MEMCPY, cudaMemcpyHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMCPY_H2D, cudaMemcpyH2DHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMCPY_D2H, cudaMemcpyD2HHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMCPY_D2D, cudaMemcpyD2DHandle, _dataWorker, this); + RegisterHandler(NEW_ITERATION_REQ, NewIterHandle, _dataWorker, this); + // RegisterHandler(CUDA_MEMCPY_ASYNC, cudaMemcpyAsyncHandle, _dataWorker, this); //todo + RegisterHandler(CUDA_MEMCPY_ASYNC_H2D, cudaMemcpyAsyncH2DHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMCPY_ASYNC_D2H, cudaMemcpyAsyncD2HHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMCPY_ASYNC_D2D, cudaMemcpyAsyncD2DHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMCPY_TO_SYMBOL, cudaMemcpyToSymbolHandle, _dataWorker, this); + RegisterHandler(CUDA_DEVICE_SYNCHRONIZE, cudaDeviceSynchronizeHandle, _dataWorker, this); + RegisterHandler(CUDA_GET_DEVICE, cudaGetDeviceHandle, _dataWorker, this); + RegisterHandler(CUDA_GET_DEVICE_COUNT, cudaGetDeviceCountHandle, _dataWorker, this); + RegisterHandler(CUDA_SET_MAIN_DEVICE, cudaSetMainDeviceHandle, _dataWorker, this); + RegisterHandler(CUDA_SET_DEVICE, cudaSetDeviceHandle, _dataWorker, this); + RegisterHandler(CUDA_GET_DEVICE_PROPERTIES, cudaGetDevicePropertiesHandle, _dataWorker, this); + RegisterHandler(CUDA_DEVICE_GET_ATTRIBUTE, cudaDeviceGetAttributeHandle, _dataWorker, this); + RegisterHandler(CUDA_EVENT_CREATE, cudaEventCreateHandle, _dataWorker, this); + RegisterHandler(CUDA_EVENT_CREATE_WITH_FLAGS, cudaEventCreateWithFlagsHandle, _dataWorker, this); + RegisterHandler(CUDA_EVENT_RECORD, cudaEventRecordHandle, _dataWorker, this); + RegisterHandler(CUDA_EVENT_QUERY, cudaEventQueryHandle, _dataWorker, this); + RegisterHandler(CUDA_EVENT_DESTROY, cudaEventDestroyHandle, _dataWorker, this); + RegisterHandler(CUDA_EVENT_ELAPSED_TIME, cudaEventElapsedTimeHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_DESTROY, cudaStreamDestroyHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_CREATE, cudaStreamCreateHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_CREATE_WITH_FLAGS, cudaStreamCreateWithFlagsHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_CREATE_WITH_PRIORITY, cudaStreamCreateWithPriorityHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_WAIT_EVENT, cudaStreamWaitEventHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_SYNCHRONIZE, cudaStreamSynchronizeHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_IS_CAPTURING, cudaStreamIsCapturingHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_GET_CAPTURE_INFO, cudaStreamGetCaptureInfoHandle, _dataWorker, this); + RegisterHandler(CUDA_OCCUPANCY_MAX_ACTIVE_BLOCKS_PER_MULTIPROCESSOR, cudaOccupancyMaxActiveBlocksPerMultiprocessorHandle, _dataWorker, this); + RegisterHandler(CUDA_OCCUPANCY_MAX_ACTIVE_BLOCKS_PER_MULTIPROCESSOR_WITH_FLAGS, cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlagsHandle, _dataWorker, this); + + + RegisterHandler(CUBLAS_CREATE_V2, cublasCreate_v2Handle, _dataWorker, this); + RegisterHandler(CUBLAS_SGEMM_V2, cublasSgemm_v2Handle, _dataWorker, this); + RegisterHandler(CUBLAS_SGEMM_STRIDED_BATCHED, cublasSgemmStridedBatchedHandle, _dataWorker, this); + RegisterHandler(CUBLAS_DESTROY_V2, cublasDestroy_v2Handle, _dataWorker, this); + RegisterHandler(CUBLAS_SET_STREAM_V2, cublasSetStream_v2Handle, _dataWorker, this); + RegisterHandler(CUBLAS_SET_WORKSPACE_V2, cublasSetWorkspace_v2Handle, _dataWorker, this); + RegisterHandler(CUBLAS_SET_MATH_MODE, cublasSetMathModeHandle, _dataWorker, this); + RegisterHandler(CUBLAS_GET_MATH_MODE, cublasGetMathModeHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_CREATE, cublasLtCreateHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_DESTROY, cublasLtDestroyHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULDESC_CREATE, cublasLtMatmulDescCreateHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULDESC_DESTROY, cublasLtMatmulDescDestroyHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULDESC_SETATTRIBUTE, cublasLtMatmulDescSetAttributeHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATRIX_LAYOUT_CREATE, cublasLtMatrixLayoutCreateHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATRIX_LAYOUT_DESTROY, cublasLtMatrixLayoutDestroyHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATRIX_LAYOUT_SETATTRIBUTE, cublasLtMatrixLayoutSetAttributeHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULPREFERENCE_CREATE, cublasLtMatmulPreferenceCreateHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULPREFERENCE_DESTROY, cublasLtMatmulPreferenceDestroyHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULPREFERENCE_SETATTRIBUTE, cublasLtMatmulPreferenceSetAttributeHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULALGO_GETHEURISTIC, cublasLtMatmulAlgoGetHeuristicHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMUL, cublasLtMatmulHandle, _dataWorker, this); + + + RegisterHandler(CUDNN_CREATE, cudnnCreateHandle, _dataWorker, this); + RegisterHandler(CUDNN_DESTROY, cudnnDestroyHandle, _dataWorker, this); + RegisterHandler(CUDNN_CREATE_TENSOR_DESCRIPTOR, cudnnCreateTensorDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_DESTROY_TENSOR_DESCRIPTOR, cudnnDestroyTensorDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_TENSOR_SIZE_IN_BYTES, cudnnGetTensorSizeInBytesHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_TENSOR_4D_DESCRIPTOR, cudnnSetTensor4dDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_TENSOR_ND_DESCRIPTOR, cudnnSetTensorNdDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_TENSOR_ND_DESCRIPTOR_EX, cudnnSetTensorNdDescriptorExHandle, _dataWorker, this); + RegisterHandler(CUDNN_CREATE_TENSOR_TRANSFORM_DESCRIPTOR, cudnnCreateTensorTransformDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_TENSOR_TRANSFORM_DESCRIPTOR, cudnnSetTensorTransformDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_DESTROY_TENSOR_TRANSFORM_DESCRIPTOR, cudnnDestroyTensorTransformDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_INIT_TRANSFORM_DEST, cudnnInitTransformDestHandle, _dataWorker, this); + RegisterHandler(CUDNN_TRANSFORM_TENSOR_EX, cudnnTransformTensorExHandle, _dataWorker, this); + RegisterHandler(CUDNN_TRANSFORM_FILTER, cudnnTransformFilterHandle, _dataWorker, this); + RegisterHandler(CUDNN_CREATE_FILTER_DESCRIPTOR, cudnnCreateFilterDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_FILTER_ND_DESCRIPTOR, cudnnSetFilterNdDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_DESTROY_FILTER_DESCRIPTOR, cudnnDestroyFilterDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_FILTER_SIZE_IN_BYTES, cudnnGetFilterSizeInBytesHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_FOLDED_CONV_BACKWARD_DATA_DESCRIPTORS, cudnnGetFoldedConvBackwardDataDescriptorsHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_STREAM, cudnnSetStreamHandle, _dataWorker, this); + RegisterHandler(CUDNN_BATCH_NORMALIZATION_BACKWARD_EX, cudnnBatchNormalizationBackwardExHandle, _dataWorker, this); + RegisterHandler(CUDNN_BATCH_NORMALIZATION_FORWARD_TRAINING_EX, cudnnBatchNormalizationForwardTrainingExHandle, _dataWorker, this); + RegisterHandler(CUDNN_BATCH_NORMALIZATION_FORWARD_INFERENCE, cudnnBatchNormalizationForwardInferenceHandle, _dataWorker, this); + RegisterHandler(CUDNN_BACKEND_CREATE_DESCRIPTOR, cudnnBackendCreateDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_BACKEND_DESTROY_DESCRIPTOR, cudnnBackendDestroyDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_BACKEND_SET_ATTRIBUTE, cudnnBackendSetAttributeHandle, _dataWorker, this); + RegisterHandler(CUDNN_BACKEND_GET_ATTRIBUTE, cudnnBackendGetAttributeHandle, _dataWorker, this); + RegisterHandler(CUDNN_BACKEND_EXECUTE, cudnnBackendExecuteHandle, _dataWorker, this); + RegisterHandler(CUDNN_BACKEND_FINALIZE, cudnnBackendFinalizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_BATCH_NORMALIZATION_BACKWARD_EX_WORKSPACE_SIZE, cudnnGetBatchNormalizationBackwardExWorkspaceSizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_BATCH_NORMALIZATION_FORWARD_TRAINING_EX_WORKSPACE_SIZE, cudnnGetBatchNormalizationForwardTrainingExWorkspaceSizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_BATCH_NORMALIZATION_TRAINING_EX_RESERVE_SPACE_SIZE, cudnnGetBatchNormalizationTrainingExReserveSpaceSizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_CREATE_CONVOLUTION_DESCRIPTOR, cudnnCreateConvolutionDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_DESTROY_CONVOLUTION_DESCRIPTOR, cudnnDestroyConvolutionDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_CONVOLUTION_GROUP_COUNT, cudnnSetConvolutionGroupCountHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_CONVOLUTION_MATH_TYPE, cudnnSetConvolutionMathTypeHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_CONVOLUTION_ND_DESCRIPTOR, cudnnSetConvolutionNdDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_CONVOLUTION_REORDER_TYPE, cudnnSetConvolutionReorderTypeHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_CONVOLUTION_FORWARD_ALGORITHM_V7, cudnnGetConvolutionForwardAlgorithm_v7Handle, _dataWorker, this); + RegisterHandler(CUDNN_GET_CONVOLUTION_BACKWARD_FILTER_ALGORITHM_V7, cudnnGetConvolutionBackwardFilterAlgorithm_v7Handle, _dataWorker, this); + RegisterHandler(CUDNN_GET_CONVOLUTION_BACKWARD_DATA_ALGORITHM_V7, cudnnGetConvolutionBackwardDataAlgorithm_v7Handle, _dataWorker, this); + RegisterHandler(CUDNN_GET_CONVOLUTION_FORWARD_WORKSPACE_SIZE, cudnnGetConvolutionForwardWorkspaceSizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_CONVOLUTION_FORWARD, cudnnConvolutionForwardHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_CONVOLUTION_BACKWARD_DATA_WORKSPACE_SIZE, cudnnGetConvolutionBackwardDataWorkspaceSizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_CONVOLUTION_BACKWARD_FILTER, cudnnConvolutionBackwardFilterHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_CONVOLUTION_BACKWARD_FILTER_WORKSPACE_SIZE, cudnnGetConvolutionBackwardFilterWorkspaceSizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_CONVOLUTION_BACKWARD_DATA, cudnnConvolutionBackwardDataHandle, _dataWorker, this); + RegisterHandler(NCCL_GROUP_START, ncclGroupStartHandle, _dataWorker, this); + RegisterHandler(NCCL_GROUP_END, ncclGroupEndHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_INIT_RANK, ncclCommInitRankHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_DESTROY, ncclCommDestroyHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_GET_ASYNC_ERROR, ncclCommGetAsyncErrorHandle, _dataWorker, this); + RegisterHandler(NCCL_GET_UNIQUE_ID, ncclGetUniqueIdHandle, _dataWorker, this); + RegisterHandler(NCCL_GET_VERSION, ncclGetVersionHandle, _dataWorker, this); + RegisterHandler(NCCL_ALL_REDUCE, ncclAllReduceHandle, _dataWorker, this); + RegisterHandler(NCCL_REDUCE, ncclReduceHandle, _dataWorker, this); + RegisterHandler(NCCL_REDUCE_SCATTER, ncclReduceScatterHandle, _dataWorker, this); + RegisterHandler(NCCL_ALL_GATHER, ncclAllGatherHandle, _dataWorker, this); + RegisterHandler(NCCL_BROADCAST, ncclBroadcastHandle, _dataWorker, this); + RegisterHandler(NCCL_SEND, ncclSendHandle, _dataWorker, this); + RegisterHandler(NCCL_RECV, ncclRecvHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_COUNT, ncclCommCountHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_USER_RANK, ncclCommUserRankHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_CU_DEVICE, ncclCommCuDeviceHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_ABORT, ncclCommAbortHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_INIT_ALL, ncclCommInitAllHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_INIT_RANK_CONFIG, ncclCommInitRankConfigHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_SPLIT, ncclCommSplitHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_FINALIZE, ncclCommFinalizeHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_REGISTER, ncclCommRegisterHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_DEREGISTER, ncclCommDeregisterHandle, _dataWorker, this); + RegisterHandler(NCCL_MEM_ALLOC, ncclMemAllocHandle, _dataWorker, this); + RegisterHandler(NCCL_MEM_FREE, ncclMemFreeHandle, _dataWorker, this); + RegisterHandler(NCCL_RED_OP_CREATE_PRE_MUL_SUM, ncclRedOpCreatePreMulSumHandle, _dataWorker, this); + RegisterHandler(NCCL_RED_OP_DESTROY, ncclRedOpDestroyHandle, _dataWorker, this); + + while (!connStatus_.isClosed) { + uint res = ucp_worker_progress(_dataWorker); +#ifdef GV_Scheduler + if(res != 0 && CliDev_ != -1){ + sche->enqueue(clientID_, CliDev_); +#ifdef GV_Monitor + if(getDeviceFlag == 0){ + std::string message = "client_runjob:"; + sche->send_message(message,CliDev_); + getDeviceFlag = 1; + } +#endif + } +#endif + +#ifdef GV_MSGHANDLER + if(res != 0 && CliDev_ != -1 && replayFlag_ && !createThreadFlag){ + msghandler->replay_flag = true; + createThreadFlag = true; + boost::thread_attributes attrs; + attrs.set_stack_size(THREAD_STACK_SIZE); + MSGRecvThread_ = new boost::thread(attrs, boost::bind(&MsgHandler::aysnc_receive_message, msghandler)); + MSGRecvThread_->detach(); + } + MSGStopFlag_ = msghandler->stop_flag; + +#endif + +#ifdef GV_eScheduler + if(res != 0 && CliDev_ != -1){ + esche->enqueue(clientID_, CliDev_, priority_); + } +#endif + + } + + // this->~ServerEndpoint(); + delete this; +} + +DEFINE_SERVER_AM_CALLBACK(NewIterHandle) { + const char* myName = "NewIteration"; + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + size_t tensorNum = reqBuf.Pop(); + TensorInfo_t* tensors = reqBuf.AssignAddrForAll(); + serverEp->trainTensors_.clear(); + size_t totalTensorSize = 0; + for (size_t i = 0; i < tensorNum; i++) { + serverEp->trainTensors_.push_back(tensors[i]); + totalTensorSize += tensors[i].size; + } + + serverEp->curIter_ ++; + tool::Logging(LOG_DEBUG, myName, "client#%llu starts the iteration#%zu.\n", serverEp->clientID_, serverEp->curIter_); + if (serverEp->curIter_ > 1 && serverEp->_cuInfoMap.isFirstIter) { + serverEp->_cuInfoMap.isFirstIter = false; + } + +#ifdef GV_eScheduler + serverEp->esche->cal_add_It(serverEp->clientID_); +#endif + +#ifdef GV_BACKUP + if (serverEp->curIter_ > 0 && serverEp->curIter_ % BACKUP_PERIOD == 0 + && !serverEp->recoveryFlag_) { // avoid backup in replay mode + auto start_back = std::chrono::high_resolution_clock::now(); + serverEp->Backup2Memory(); + auto end_back = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration_back = end_back - start_back; + tool::Logging(myName, "backup total time=%f s\n", duration_back.count()); + serverEp->ckptCnt++; // debug + } + + if (serverEp->curIter_ >= BACKUP_PERIOD && !serverEp->recoveryFlag_) { + // serverEp->Backup2Storage(serverEp->curIter_); + auto start = std::chrono::high_resolution_clock::now(); + if (!serverEp->trainTensorBackup_.CheckStart()) { + serverEp->trainTensorBackup_.Start(boost::bind(&ServerEndpoint::BackupTrainTensors2Storage, serverEp)); + } + serverEp->trainTensorBackup_.Notify(); + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration = end - start; + tool::Logging(LOG_DEBUG, myName, "submit the backup task for iteration#%zu in %f seconds.\n", serverEp->curIter_, duration.count()); + } + else if (serverEp->recoveryFlag_) { + serverEp->LoadFromStorage(serverEp->curIter_ - 1); // curIter_ starts from 1 while restoring + } + + +/* + if (serverEp->curIter_ == BACKUP_PERIOD && !serverEp->recoveryFlag_) { + cudaError_t exit_code = cudaMallocHost(&serverEp->backupMemcpyBuffer_, totalTensorSize * BACKUP_PERIOD); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMallocHost failed for backup memcpy buffer: %s\n", cudaGetErrorString(exit_code)); + exit(EXIT_FAILURE); + } + } + if (serverEp->backupMemcpyBuffer_ && !serverEp->recoveryFlag_) { + auto start = std::chrono::high_resolution_clock::now(); + size_t offset = (serverEp->curIter_ % BACKUP_PERIOD) * totalTensorSize; // buffer starts from curIter_ = 0 + for (size_t i = 0; i < tensorNum; i++) { + int streamIdx = i % BACKUP_STREAM_NUM; + // cudaError_t exit_code = cudaMemcpy((uint8_t*)serverEp->backupMemcpyBuffer_ + offset, serverEp->GetDevPtr((uint64_t)tensors[i].devPtr), tensors[i].size, cudaMemcpyDeviceToHost); + cudaError_t exit_code = cudaMemcpyAsync((uint8_t*)serverEp->backupMemcpyBuffer_ + offset, serverEp->GetDevPtr((uint64_t)tensors[i].devPtr), tensors[i].size, cudaMemcpyDeviceToHost, serverEp->streamList_[streamIdx]); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpy failed for tensor[%zu](%p) with size %zu: %s\n", i, tensors[i].devPtr, tensors[i].size, cudaGetErrorString(exit_code)); + exit(EXIT_FAILURE); + } + else { + tool::Logging(LOG_DEBUG, myName, "cudaMemcpy for tensor[%zu](%p) with size %zu in stream %p.\n", i, tensors[i].devPtr, tensors[i].size, serverEp->streamList_[streamIdx]); + } + + offset += tensors[i].size; + } + auto end = std::chrono::high_resolution_clock::now(); + tool::Logging(LOG_DEBUG, myName, "cudaMemcpy for %zu bytes takes %f seconds\n", totalTensorSize, std::chrono::duration(end - start).count()); + } + if (serverEp->recoveryFlag_) { + size_t offset = ((serverEp->curIter_ - 1) % BACKUP_PERIOD) * totalTensorSize; // curIter_ starts from 1 + for (size_t i = 0; i < tensorNum; i++) { + cudaError_t exit_code = cudaMemcpyAsync(serverEp->GetDevPtr((uint64_t)tensors[i].devPtr), (uint8_t*)serverEp->backupMemcpyBuffer_ + offset, tensors[i].size, cudaMemcpyHostToDevice, serverEp->defaultStream_); + // cudaMemcpy(serverEp->GetDevPtr((uint64_t)tensors[i].devPtr), (uint8_t*)serverEp->backupMemcpyBuffer_ + offset, tensors[i].size, cudaMemcpyHostToDevice); + offset += tensors[i].size; + } + // cudaStreamSynchronize(serverEp->defaultStream_); + tool::Logging(LOG_DEBUG, myName, "recovery the tensors for iteration#%zu in stream %p.\n", serverEp->curIter_, serverEp->defaultStream_); + } +*/ + +#endif // GV_BACKUP + + return UCS_OK; +} + +void ServerEndpoint::CreateServerEp(ucp_conn_request_h conn_request, void *arg) { + const char* myName = "CreateServerEp"; + UCPConnection* connectionObj = (UCPConnection*)arg; + UCPConnection_t conn; + + char* ip_str = (char*)malloc(IP_STRING_LEN); + char* port_str = (char*)malloc(PORT_STRING_LEN); + ucp_conn_request_attr_t attr; + attr.field_mask = UCP_CONN_REQUEST_ATTR_FIELD_CLIENT_ADDR | UCP_CONN_REQUEST_ATTR_FIELD_CLIENT_ID; + ucs_status_t status = ucp_conn_request_query(conn_request, &attr); + if (status == UCS_OK) { // get the client address from the connection request + tool::GetIpStrFromSockaddr(&attr.client_address, ip_str, IP_STRING_LEN); + tool::GetPortStrFromSockaddr(&attr.client_address, port_str, PORT_STRING_LEN); + tool::Logging(LOG_INFO, myName, "recv a connection request(%s:%s) from client#%llu.\n", ip_str, port_str, attr.client_id); + conn.client_id = attr.client_id; + conn.client_ip = ip_str; + conn.client_port = port_str; + conn.conn_request = conn_request; + + } else if (status != UCS_ERR_UNSUPPORTED) { + tool::Logging(LOG_ERROR, myName, "failed to query the connection request (%s)\n", ucs_status_string(status)); // todo: UCS_ERR_UNSUPPORTED + exit(EXIT_FAILURE); + } + + ServerEndpoint* serverEndPointObj = new ServerEndpoint(connectionObj->CreateWorker(), conn); + + boost::thread_attributes attrs; + attrs.set_stack_size(THREAD_STACK_SIZE); + boost::thread* thTmp = new boost::thread(attrs, boost::bind(&ServerEndpoint::NewRun, serverEndPointObj)); + thTmp->detach(); + delete thTmp; +} + + +/** + * Close UCP endpoint. + */ +void ServerEndpoint::CloseEp(uint64_t flags) { + ucp_request_param_t param; + ucs_status_t status; + void *close_req; + + param.op_attr_mask = UCP_OP_ATTR_FIELD_FLAGS; + param.flags = flags; + close_req = ucp_ep_close_nbx(_clientEp, ¶m); + if (UCS_PTR_IS_PTR(close_req)) { + do { + ucp_worker_progress(_dataWorker); + status = ucp_request_check_status(close_req); + } while (status == UCS_INPROGRESS); + ucp_request_free(close_req); + } else { + status = UCS_PTR_STATUS(close_req); + } + + if (status != UCS_OK) { + fprintf(stderr, "failed to close ep %p: %s\n", (void*)_clientEp, + ucs_status_string(status)); + } +} + +ServerEndpoint::~ServerEndpoint() { + CloseEp(UCP_EP_CLOSE_MODE_FLUSH); + ucp_worker_destroy(_dataWorker); + //free(_connectReq); + //ucp_listener_reject(_serverConnection.listener, _serverConnection.conn_request); + // if (UCS_PTR_IS_PTR(_connectReq)) { + // ucp_request_free(_connectReq); + // } //! In multi-threaded version, it may cause the error + +#ifdef GV_Scheduler + if(CliDev_ != -1){ + sche->free_jobs(clientID_, CliDev_); +#ifdef GV_Monitor + std::string message = "client_stop:"; + sche->send_message(message,CliDev_); +#endif + } +#endif + +#ifdef GV_eScheduler + if(CliDev_ != -1){ + esche->free_jobs(clientID_, CliDev_); + } + delete esche; +#endif + + + for (int i = 0; i < 3; i++) { + delete shmQueSizes_[i]; + delete shmQueues_[i]; + } + +#ifdef GV_MSGHANDLER + // MSGRecvThread_->join(); + delete msghandler; + // MSGRecvThread_->interrupt(); + // delete MSGRecvThread_; +#endif + + +#ifdef GV_Scheduler +#ifdef GV_Monitor + bool isClosed = sche->ready_to_del(); + if(isClosed){ + delete sche; + } +#ifndef GV_Monitor + delete sche; +#endif +#endif + +#endif + // delete[] blockList_; + + + __cudaUnregisterFatBinaryHandle(); + + tool::Logging(LOG_INFO, myName_, "close the Server endpoint object.\n"); + // printf("zwx mapFatBinary=%p, mapFatBinHandle2CuModule=%p\n", _cuInfoMap.mapFatBinary, _cuInfoMap.mapFatBinHandle2CuModule); + free(myName_); + free(clientIP_); + free(clientPort_); +} + +void ServerEndpoint::SendResponse(RequestIOV* reqBuffer, const ucp_ep_h* ep, ucs_memory_type_t memType) { + SendData(reqBuffer->GetIOVs(), reqBuffer->GetNum(), + reqBuffer->GetHeaders(), reqBuffer->GetHeaderSize(), + (reqBuffer->GetThreadID() << RECV_AM_SHIFT_BIT) + reqBuffer->GetRequestType(), + &_dataWorker, (ucp_ep_h*)ep, false, memType); +} + +void ServerEndpoint::SendStatus(int status) { + RequestIOV resBuf; + resBuf.PushRequestType(SERVER_STATUS); + resBuf.Push(status); + int dummy = 0; + resBuf.PushThreadID(dummy); + SendData(resBuf.GetIOVs(), resBuf.GetNum(), + resBuf.GetHeaders(), resBuf.GetHeaderSize(), + resBuf.GetRequestType(), &_dataWorker, &_clientEp); +} + +void ServerEndpoint::UpdateStream(cudaStream_t stream, bool isCommStream) { + if (isCommStream) { + commStream_ = stream; + } else { + defaultStream_ = stream; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/Design.png b/GPU-Virtual-Service/gpu-remoting/src/storage/Design.png new file mode 100644 index 0000000..af3d11a Binary files /dev/null and b/GPU-Virtual-Service/gpu-remoting/src/storage/Design.png differ diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/PyShmqueue.py b/GPU-Virtual-Service/gpu-remoting/src/storage/PyShmqueue.py new file mode 100644 index 0000000..60ba776 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/PyShmqueue.py @@ -0,0 +1,206 @@ +import copy +import ctypes +import os +import time +import numpy as np +from ctypes import * +CACHE_LINE_SIZE = 64 +# Define QueueModule +ONE_READ_ONE_WRITE = 0 +ONE_READ_MUL_WRITE = 1 +MUL_READ_ONE_WRITE = 2 +MUL_READ_MUL_WRITE = 3 + +SHAR_KEY_MODEL = 100010 +SHAR_KEY_TRAIN = 100020 +SHAR_KEY_VAL = 100030 +SHAR_KEY_TEST = 100040 + + +class stMemTrunk(Structure): + _fields_ = [("m_iBegin", ctypes.c_uint), + ("__cache_padding1__", ctypes.c_char * CACHE_LINE_SIZE), + ("m_iEnd", ctypes.c_uint), + ("__cache_padding2__", ctypes.c_char * CACHE_LINE_SIZE), + ("m_iShmKey", ctypes.c_int), + ("__cache_padding3__", ctypes.c_char * CACHE_LINE_SIZE), + ("m_iSize", ctypes.c_uint), + ("__cache_padding4__", ctypes.c_char * CACHE_LINE_SIZE), + ("m_iShmId", ctypes.c_int), + ("__cache_padding5__", ctypes.c_char * CACHE_LINE_SIZE), + ("m_eQueueModule", ctypes.c_int)] + +class CShmRWlock(ctypes.Structure): + _fields_ = [("m_iSemID", ctypes.c_int), + ("m_iSemKey", ctypes.c_int)] + +class CMessageQueue(ctypes.Structure): + _fields_ = [("m_stMemTrunk", ctypes.POINTER(stMemTrunk)), + ("m_pBeginLock", ctypes.POINTER(CShmRWlock)), + ("m_pEndLock", ctypes.POINTER(CShmRWlock)), + ("m_pQueueAddr", ctypes.POINTER(ctypes.c_ubyte)), + ("m_pShm", ctypes.c_void_p)] + +print("[Python]: Start loading libshmmqueue.so...") +libLoad = ctypes.cdll.LoadLibrary +try: + module_root_path = os.path.dirname(__file__) + parent_of_parent_path = os.path.dirname(os.path.dirname(module_root_path)) + target_lib_path = os.path.join(parent_of_parent_path, "out", "lib64", "libshmmqueue.so") + share = libLoad(target_lib_path) + print("[Python]: Loading libshmmqueue.so successful!") + # print(share.IsPowerOfTwo(3)) + # Define the function signatures + share.CreateInstance.argtypes = [ctypes.c_int, ctypes.c_size_t, ctypes.c_int] + share.CreateInstance.restype = ctypes.POINTER(CMessageQueue) + + share.GetInstance.argtypes = [ctypes.c_int, ctypes.c_size_t, ctypes.c_int] + share.GetInstance.restype = ctypes.POINTER(CMessageQueue) + + share.DestroyInstance.argtypes = [ctypes.POINTER(CMessageQueue)] + share.DestroyInstance.restype = None + + share.SendMessage.argtypes = [ctypes.POINTER(CMessageQueue), ctypes.POINTER(ctypes.c_ubyte), ctypes.c_size_t] + share.SendMessage.restype = ctypes.c_int + + share.GetMessage.argtypes = [ctypes.POINTER(CMessageQueue), ctypes.POINTER(ctypes.c_ubyte)] + share.GetMessage.restype = ctypes.c_int + + share.ReadMessage.argtypes = [ctypes.POINTER(CMessageQueue), ctypes.POINTER(ctypes.c_ubyte)] + share.ReadMessage.restype = ctypes.c_int + + share.ReadHeadMessage.argtypes = [ctypes.POINTER(CMessageQueue), ctypes.POINTER(ctypes.c_ubyte)] + share.ReadHeadMessage.restype = ctypes.c_int + + share.DeleteHeadMessage.argtypes = [ctypes.POINTER(CMessageQueue)] + share.DeleteHeadMessage.restype = ctypes.c_int + + share.PrintTrunk.argtypes = [ctypes.POINTER(CMessageQueue)] + share.PrintTrunk.restype = None + + share.GetFreeSize.argtypes = [ctypes.POINTER(CMessageQueue)] + share.GetFreeSize.restype = ctypes.c_uint + + share.GetDataSize.argtypes = [ctypes.POINTER(CMessageQueue)] + share.GetDataSize.restype = ctypes.c_uint + + share.GetQueueLength.argtypes = [ctypes.POINTER(CMessageQueue)] + share.GetQueueLength.restype = ctypes.c_uint + + share.ClearQueue.argtypes = [ctypes.POINTER(CMessageQueue)] + share.ClearQueue.restype = None + + share.InitLock.argtypes = [ctypes.POINTER(CMessageQueue)] + share.InitLock.restype = None + + share.IsBeginLock.argtypes = [ctypes.POINTER(CMessageQueue)] + share.IsBeginLock.restype = ctypes.c_int + + share.IsEndLock.argtypes = [ctypes.POINTER(CMessageQueue)] + share.IsEndLock.restype = ctypes.c_int + + share.CreateShareMem.argtypes = [ctypes.c_int, ctypes.c_long, ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int)] + share.CreateShareMem.restype = ctypes.POINTER(ctypes.c_ubyte) + + share.DestroyShareMem.argtypes = [ctypes.c_void_p, ctypes.c_int] + share.DestroyShareMem.restype = ctypes.c_int + + share.IsPowerOfTwo.argtypes = [ctypes.c_size_t] + share.IsPowerOfTwo.restype = ctypes.c_int + + share.Fls.argtypes = [ctypes.c_size_t] + share.Fls.restype = ctypes.c_int + + share.RoundupPowofTwo.argtypes = [ctypes.c_size_t] + share.RoundupPowofTwo.restype = ctypes.c_size_t + # share.get_share_body_address.restype = ctypes.POINTER(ctypes.c_uint8) +except Exception as e: + print("[Python]: Failed to load shared memory dynamic library! Probably due to not placing libshmmqueue.so in out/lib64") + print("[Python]: Detailed error messages-" + str(e)) + exit(1) + + + + +class ShareMemoryQueue: + def __init__(self, shmkey, queuesize, queueModule, CREATE_FLAG): + self.stop_blocking = False + self.shmkey = shmkey + # get existed ShareMemoryQueue + if(CREATE_FLAG): + self.message_queue_ptr = share.CreateInstance(shmkey, queuesize, queueModule) + else: + self.message_queue_ptr = share.GetInstance(shmkey, queuesize, queueModule) + + def write(self, batch): + data_ptr = batch.ctypes.data + bytes_ptr = ctypes.cast(data_ptr, ctypes.POINTER(ctypes.c_ubyte)) + error_code = share.SendMessage(self.message_queue_ptr, bytes_ptr, batch.nbytes) + print_flag = False + # self.stop_blocking = False + while error_code !=0: + if self.stop_blocking == True: + break + if not print_flag: + print(f"[shmkey: {self.shmkey}] write failed, error_code is {error_code}, Write blocking...") + print_flag = True + error_code = share.SendMessage(self.message_queue_ptr, bytes_ptr, batch.nbytes) + # print(f"write #batch_idx-{batch_idx} success") + # bytes_batch = batch.tobytes() + # data_ptr = ctypes.cast(ctypes.create_string_buffer(bytes_batch), ctypes.POINTER(ctypes.c_ubyte)) + # res = share.SendMessage(self.message_queue_ptr, data_ptr, len(bytes_batch)) + return error_code + + def get(self): + buffer_size = 100 # Replace with the desired buffer size + buffer = np.zeros(buffer_size, dtype=np.uint8) + result = share.GetMessage(self.message_queue_ptr, buffer.ctypes.data_as(POINTER(c_ubyte))) + print("Received data: ", buffer[:result]) + + def read(self, buffer_size): + # buffer_size = 100 # Replace with the desired buffer size + buffer = np.zeros(buffer_size, dtype=np.uint8) + result = share.ReadMessage(self.message_queue_ptr, buffer.ctypes.data_as(POINTER(c_ubyte))) + # print("Read data: ", buffer) + + def ready(self): + self.stop_blocking = False + + def clear(self): + self.stop_blocking = True + share.ClearQueue(self.message_queue_ptr) + print(f"[shmkey: {self.shmkey}] clear queue, stop blocking = {self.stop_blocking}") + + def GetDataSize(self): + return share.GetDataSize(self.message_queue_ptr) + + def PrintTrunk(self): + share.PrintTrunk(self.message_queue_ptr) + + def getQueueLength(self): + share.GetQueueLength(self.message_queue_ptr) + + # def __del__(self): + # share.DestroyInstance(self.message_queue_ptr) + # print("Destroy ShareMemoryQueue") + + +def test(): + shmkey = 100010 + queuesize = 10240 + queueModule = ONE_READ_ONE_WRITE + shmqueue = ShareMemoryQueue(shmkey, 10 * 2, queueModule, 1) + + # data = np.array([1, 2, 3, 4, 5 ,6, 7], dtype=np.uint8) # Assuming you have some data to write + sequence_array = np.arange(10, dtype=np.uint8) + print("write data: ", sequence_array) + print("len: ", len(sequence_array)) + for i in range(10): + error_code = shmqueue.write(sequence_array, i) + while error_code != 0: + error_code = shmqueue.write(sequence_array, i) + shmqueue.PrintTrunk() + + + # print(shmqueue.getQueueLength()) +# test() \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/code_overview.png b/GPU-Virtual-Service/gpu-remoting/src/storage/code_overview.png new file mode 100644 index 0000000..03cc31b Binary files /dev/null and b/GPU-Virtual-Service/gpu-remoting/src/storage/code_overview.png differ diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/common.py b/GPU-Virtual-Service/gpu-remoting/src/storage/common.py new file mode 100644 index 0000000..1220ea1 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/common.py @@ -0,0 +1,104 @@ +import json +import torch +import ctypes +import logging +import pickle +import struct + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +logging.getLogger("PIL.TiffImagePlugin").setLevel(51) + +EPOCH_STOP = 0 +DATALOADER_REQ = 1 +EPOCH_REQ = 2 + +class DataloaderInfo: + def __init__(self, train=True, remote_dataset_path=None, regular_dataset=None, batch_size=1, shuffle=False, sampler=None, batch_sampler=None, collate_fn=None, drop_last=False, preprocess=None, generator=None, num_workers=0, other_dataloader=None, distributed=False, seed=0, rank=0, world_size=1, sample_drop_last=False): + self.train = train + self.remote_dataset_path = remote_dataset_path + self.regular_dataset = regular_dataset + self.batch_size = batch_size + self.shuffle = shuffle + self.sampler = sampler + self.batch_sampler = batch_sampler + self.collate_fn = collate_fn + self.drop_last = drop_last + self.preprocess = preprocess + self.generator = generator + self.num_workers = num_workers + self.other_dataloader = other_dataloader + self.distributed = distributed + self.seed = seed + self.rank = rank + self.world_size = world_size + self.sample_drop_last = sample_drop_last + + def __str__(self): + return f"\nDataloaderInfo(train: {self.train}, remote_dataset_path: {self.remote_dataset_path}, batch_size: {self.batch_size}, shuffle: {self.shuffle}, sampler: {self.sampler}, batch_sampler: {self.batch_sampler}, collate_fn: {self.collate_fn}, drop_last: {self.drop_last}, preprocess: {self.preprocess}), generator: {self.generator}, num_workes: {self.num_workers},\ndistributed: {self.distributed}, seed: {self.seed}, rank: {self.rank}, world_size: {self.world_size}, sample_drop_last: {self.sample_drop_last},\nregular_dataset: {self.regular_dataset},\nother_dataloader: {self.other_dataloader}" + +class BatchInfo: + def __init__(self, train, batch_num, data_dtype=None, label_dtype=None, first_data_shape=None, first_label_shape=None, last_data_shape=None, last_label_shape=None): + self.train = train + self.batch_num = batch_num + self.data_dtype = data_dtype + self.label_dtype = label_dtype + self.first_data_shape = first_data_shape + self.first_label_shape = first_label_shape + self.last_data_shape = last_data_shape + self.last_label_shape = last_label_shape + + + def __str__(self): + return f"BatchInfo(train: {self.train}, batch_num: {self.batch_num}, data_dtype: {self.data_dtype}, label_dtype: {self.label_dtype}, first_data_shape: {self.first_data_shape}, first_label_shape: {self.first_label_shape}, last_data_shape: {self.last_data_shape}, last_label_shape: {self.last_label_shape}" + +class DatasetMetadata: + def __init__(self, train, batch_num, is_iterable, first_batch_metadata, last_batch_metadata): + self.train = train + self.batch_num = batch_num + self.is_iterable = is_iterable + self.first_batch = first_batch_metadata + self.last_batch = last_batch_metadata + + def __str__(self): + return f"DatasetMetadata(train: {self.train}, batch_num: {self.batch_num}, is_iterable:{self.is_iterable} ,first_batch: {self.first_batch}, last_batch: {self.last_batch}" + + +def get_server_ep(): + with open('config.json') as f: + config = json.load(f) + server_port = config['DatasetHandlerConfig']['handlerPort_'] + server_ip = config['DatasetHandlerConfig']['handlerIp_'] + return server_ip, server_port + +def get_client_id(): + with open('config.json') as f: + config = json.load(f) + return config['ClientConfig']['clientID_'] + +def get_size_t_len(): + return ctypes.sizeof(ctypes.c_size_t) + +def send_msg(socket, msg): + msg_byte = pickle.dumps(msg) + msg_len = len(msg_byte) + length_data = struct.pack('Q', msg_len) + socket.sendall(length_data) + socket.sendall(msg_byte) + + +def recv_msg(socket): + length_data = socket.recv(get_size_t_len()) + if not length_data: + return None + msg_len = struct.unpack('Q', length_data)[0] + msg_byte = bytearray(msg_len) + view = memoryview(msg_byte) + + bytes_received = 0 + while bytes_received < msg_len: + packet_size = socket.recv_into(view[bytes_received:]) + if not packet_size: + return None + bytes_received += packet_size + + return pickle.loads(msg_byte) \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/dataFeeder.py b/GPU-Virtual-Service/gpu-remoting/src/storage/dataFeeder.py new file mode 100644 index 0000000..5e654af --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/dataFeeder.py @@ -0,0 +1,258 @@ +import threading +import numpy as np +from PyShmqueue import * +import torch +import mmap +import posix_ipc +from torch.utils.data import DataLoader +# from torchvision import datasets +import torchvision +from common import DatasetMetadata, get_size_t_len, logging, DataloaderInfo +# from datasets import load_dataset, load_from_disk +import transformers +from transformers import default_data_collator + +class DataFeeder: + def __init__(self, ident="163_1234", identNum=1, + dataloaderInfo: DataloaderInfo = None): + torch.set_num_threads(8) + self.train = dataloaderInfo.train + self.ident = ident + self.identNum = identNum + self.stop_flag = threading.Event() + self.thread = None + + self.distributed = dataloaderInfo.distributed + + if dataloaderInfo.other_dataloader is not None: + self.dataloader = dataloaderInfo.other_dataloader + else: + # from datasets import load_from_disk + # if "ImageNet" in dataloaderInfo.remote_dataset_path: + # dataset_path = "/mnt/nvme0/FlexGV_Test/ImageNet-1K/{}".format("train" if self.train else "val") + # self.dataset = torchvision.datasets.ImageFolder(dataset_path, transform=dataloaderInfo.preprocess) + # elif "GLUE" in dataloaderInfo.remote_dataset_path: + # dataset_path = "/mnt/nvme0/FlexGV_Test/GLUE/MRPC_Processed" + # local_prep_dataset = load_from_disk(dataset_path) + # self.dataset = local_prep_dataset["train"] if self.train else local_prep_dataset["validation"] + if dataloaderInfo.regular_dataset is not None: + self.dataset = dataloaderInfo.regular_dataset + else: + self.dataset = torchvision.datasets.ImageFolder(dataloaderInfo.remote_dataset_path, transform=dataloaderInfo.preprocess) + + if self.distributed: + dataloaderInfo.sampler = torch.utils.data.distributed.DistributedSampler(dataset=self.dataset, + num_replicas=dataloaderInfo.world_size, + rank=dataloaderInfo.rank, shuffle=dataloaderInfo.shuffle, + seed=dataloaderInfo.seed, drop_last=dataloaderInfo.sample_drop_last) + + self.dataloader = DataLoader(self.dataset, batch_size=dataloaderInfo.batch_size, + shuffle=(dataloaderInfo.sampler is None), + sampler=dataloaderInfo.sampler, + batch_sampler=dataloaderInfo.batch_sampler, + collate_fn=dataloaderInfo.collate_fn, + drop_last=dataloaderInfo.drop_last, + generator=dataloaderInfo.generator, num_workers=dataloaderInfo.num_workers) + self.client_batch_cnt = len(self.dataloader) + self.dataloader_iter = iter(self.dataloader) + self.is_iterable = True + self.cur_batch_index = 0 + self.queue = None + self.shm_obj = None + self.shm_desc = None + self.queue_batch_num = 2 + + def reset(self, epoch = 0): + if self.distributed: + self.dataloader.sampler.set_epoch(epoch) + self.dataloader_iter = iter(self.dataloader) + self.cur_batch_index = 0 + self.client_batch_cnt = len(self.dataloader) + logging.debug(f"DataFeeder reset: isTrain: {self.train}, client_batch_cnt: {self.client_batch_cnt}") + if self.queue is not None: + self.queue.clear() + self.queue.ready() + + + def _create_shm_queue(self, first_batch_data): + size_t_len = get_size_t_len() + # queue_size = self.queue_batch_num * (data.numel()*data.element_size()+label.numel()*label.element_size() + 2*size_t_len) + first_batch_size = 0 + if self.is_iterable == False: + first_batch_data = first_batch_data.values() + for value in first_batch_data: + # first_batch_size += (value.numel()*value.element_size() + size_t_len) + first_batch_size += (value.numel()*value.element_size() + size_t_len * 2) + queue_size = self.queue_batch_num * first_batch_size + + # record the size of shared memory queue + data_type = 1 if self.train else 2 + # self.shm_obj = posix_ipc.SharedMemory(name=("/flexgv_shm_%d_%d" % (self.client_id, data_type)), + # flags=posix_ipc.O_CREAT, size=size_t_len) + self.shm_obj = posix_ipc.SharedMemory(name=("/flexgv_shm_%s_%d" % (self.ident, data_type)), # clientID_PID_1/2 + flags=posix_ipc.O_CREAT, size=size_t_len * 3) + self.shm_desc = mmap.mmap(self.shm_obj.fd, self.shm_obj.size) + self.shm_desc.seek(0) + logging.debug(f"first_batch_size={first_batch_size}, queue_size={queue_size}, client_batch_cnt={self.client_batch_cnt}, identNum={self.identNum}, ident={self.ident}") + self.shm_desc.write(queue_size.to_bytes(size_t_len, byteorder='little')) + self.shm_desc.write(self.client_batch_cnt.to_bytes(size_t_len, byteorder='little')) + self.shm_desc.write(self.identNum.to_bytes(size_t_len, byteorder='little')) + + # get the instance of shared memory queue + self.queue = ShareMemoryQueue((self.identNum<<2) + data_type, + queue_size, ONE_READ_ONE_WRITE, 1) # 1 for creating a new queue + logging.info(f"Shared Memory Queue with size: {queue_size} created for client({self.ident}), with data type = {data_type}") + + + def get_batch_info(self, is_first=False): + # first_batch_data, first_batch_label = next(iter(self.dataloader_iter)) + first_batch_data = next(iter(self.dataloader_iter)) + if isinstance(first_batch_data, dict): + self.is_iterable = False + + first_batch_metadata, last_batch_metadata = [], [] + for col in first_batch_data: + col_metadata = {} + if self.is_iterable: + col_metadata["key"] = None + else: + col_metadata["key"] = col + col = first_batch_data[col] + col_metadata["dtype"] = col.dtype + col_metadata["shape"] = col.shape + + col_metadata_for_last = col_metadata.copy() + if self.dataloader.drop_last == False: + if hasattr(self.dataloader.sampler, 'num_samples'): # DistributedSampler has num_samples + # number of samples that the current process should process + local_dataset_size = self.dataloader.sampler.num_samples + local_last_batch_size = local_dataset_size % self.dataloader.batch_size or self.dataloader.batch_size + col_metadata_for_last["shape"] = (local_last_batch_size, *col.shape[1:]) + else: + # single GPU + last_batch_size = len(self.dataloader.dataset) % self.dataloader.batch_size or self.dataloader.batch_size + col_metadata_for_last["shape"] = (last_batch_size, *col.shape[1:]) + + first_batch_metadata.append(col_metadata) + last_batch_metadata.append(col_metadata_for_last) + + # if self.dataloader.drop_last: + # last_data_shape, last_label_shape = first_batch_data.shape, first_batch_label.shape + # else: + # last_batch_size = len(self.dataloader.dataset) % self.dataloader.batch_size or self.dataloader.batch_size + # last_data_shape = (last_batch_size, *first_batch_data.shape[1:]) + # last_label_shape = (last_batch_size, *first_batch_label.shape[1:]) + + if is_first == True: + self._create_shm_queue(first_batch_data) + else: + if self.is_iterable == False: + first_batch_data = first_batch_data.values() + for value in first_batch_data: + self.queue.write(value.cpu().numpy()) + logging.info(f"[client: {self.ident}] preload_1st_data (isTrain: {self.train}): batch_idx = {self.cur_batch_index} / {self.queue_batch_num - 1}") + self.cur_batch_index += 1 # write the first batch + self.preload_data() # write the next batches + + # return BatchInfo(self.train, len(self.dataloader), first_batch_data.dtype, first_batch_label.dtype, first_batch_data.shape, first_batch_label.shape, last_data_shape, last_label_shape) + return DatasetMetadata(self.train, len(self.dataloader), self.is_iterable, first_batch_metadata, last_batch_metadata) + + def preload_data(self): + while self.cur_batch_index < self.queue_batch_num: + data = next(self.dataloader_iter) + if self.is_iterable == False: + data = data.values() + for value in data: + self.queue.write(value.cpu().numpy()) + logging.info(f"[client: {self.ident}] preload_data (isTrain: {self.train}): batch_idx = {self.cur_batch_index} / {self.queue_batch_num - 1}") + self.cur_batch_index += 1 # write the second batch + + def load_data(self): + # data_time_list = [] + while self.cur_batch_index < self.client_batch_cnt and not self.stop_flag.is_set(): + # start_time = time.time() + data = next(self.dataloader_iter) + # data_time = time.time() - start_time + if self.is_iterable == False: + data = data.values() + for value in data: + self.queue.write(value.cpu().numpy()) + # data_time_list.append(data_time) + # for value in data: + # self.queue.read(value.numel()*value.element_size()) + # logging.info(f"data time: {data_time}") + # logging.debug(f"[client: {self.ident}] load_data (isTrain: {self.train}): batch_idx = {self.cur_batch_index} / {self.client_batch_cnt - 1}") + + if (self.cur_batch_index % 10 == 0): + logging.info(f"[client: {self.ident}] load_data (isTrain: {self.train}) for 10 batch: cur_batch_idx = {self.cur_batch_index} / {self.client_batch_cnt - 1}") + self.cur_batch_index += 1 + logging.info(f"[client: {self.ident}] load_data: dataloader(isTrain: {self.train}) has been iterated over successfully, current batch index = {self.cur_batch_index - 1}") + # logging.info(f"[client: {self.ident}] load_data: data time Avg: {sum(data_time_list)/len(data_time_list)}") + # no need to reset the dataloader, since cudaMemcpyAsync may have not been finished + + def start_load_data(self): + if self.thread is None or not self.thread.is_alive(): + self.stop_flag.clear() + self.thread = threading.Thread(target=self.load_data) + self.thread.start() + logging.info(f"[{self.ident}] started loading data(isTrain={self.train})") + else: + logging.info(f"[{self.ident}] data loading thread(isTrain={self.train}) is already running") + + def stop_load_data(self, batch_num = 0): + if self.thread is not None and self.thread.is_alive(): + if batch_num > 0: + if self.cur_batch_index < batch_num: # wait for completing the batch_num + self.client_batch_cnt = batch_num + else: # already iterated over the batch_num + self.stop_flag.set() + self.queue.clear() # avoid blocking in the writing queue + else: + self.stop_flag.set() + self.thread.join() + self.thread = None + logging.info(f"[{self.ident}] stopped loading data(isTrain={self.train})") + else: + logging.info(f"[{self.ident}] data loading thread(isTrain={self.train}) has already been stopped") + + # def feed_data(self): + # data_type = 1 if self.train else 2 + # queue_size = None + # data, label = next(self.dataloader_iter) + # if self.queue is None: + # queue_size = 2 * (data.numel()*data.element_size()+label.numel()*label.element_size() + 2*get_size_t_len()) + # data_type = 1 if self.train else 2 + # size_t_len = get_size_t_len() + # self.shm_obj = posix_ipc.SharedMemory(name=("/flexgv_shm_%d_%d" % (self.client_id, data_type)), + # flags=posix_ipc.O_CREAT, size=size_t_len) + # self.shm_desc = mmap.mmap(self.shm_obj.fd, self.shm_obj.size) + # self.shm_desc.write(queue_size.to_bytes(size_t_len, byteorder='little')) + # self.queue = ShareMemoryQueue((self.client_id<<2) + data_type, + # queue_size, ONE_READ_ONE_WRITE, 1) # 0 for not creating a new queue + # print(f"Queue with size: {queue_size} created for client{self.client_id}, with data type = {data_type}") + + # # label_bytes = label.numpy().view(np.uint8) + # # print(f"is_train: {self.train} y_bytes[:8]:{label_bytes[:8].tobytes().hex()}, y_bytes[-8:]:{label_bytes[-8:].tobytes().hex()}") + # # self.queue.PrintTrunk() + # ret = self.queue.write(data.cpu().numpy()) + # # print(f"data write ret: {ret}") + # ret = self.queue.write(label.cpu().numpy()) + # # print(f"label write ret: {ret}") + # # ret = self.queue.write(label.cpu().numpy()) + # # print(f"test third wirte: {ret}") + + # res_idx = self.batch_index + # self.batch_index += 1 + # if (self.batch_index % 50 == 0): + # print(f"HYF: feed_data: batch_idx: {self.batch_index} / {len(self.dataloader) - 1}") + # # if self.batch_index >= len(self.dataloader): + # # self.reset() + + # # return BatchInfo(self.train, res_idx, data.shape, data.dtype, label.shape, label.dtype) + + def __del__(self): + self.is_closed = False + if self.shm_obj is not None: + self.shm_desc.close() + self.shm_obj.close_fd() + # self.shm_obj.unlink() # unlinked by the CUDA handle \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/dataloaderWrapper.py b/GPU-Virtual-Service/gpu-remoting/src/storage/dataloaderWrapper.py new file mode 100644 index 0000000..3287578 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/dataloaderWrapper.py @@ -0,0 +1,220 @@ +import logging +import sys +import socket +import mmap +import posix_ipc +import torch +import os +import struct +from torch.utils.data import DataLoader, Dataset +from torch import nn +from torchvision import datasets +from torchvision.transforms import ToTensor +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from common import DataloaderInfo, BatchInfo, get_server_ep, get_client_id, logging, send_msg, recv_msg, EPOCH_REQ, EPOCH_STOP, DATALOADER_REQ +import torch.distributed as dist + +from accelerate import Accelerator +import transformers +from transformers import ( + AutoConfig, + AutoModelForSequenceClassification, + AutoTokenizer, + DataCollatorWithPadding, + PretrainedConfig, + SchedulerType, + default_data_collator, + get_scheduler, +) + + +# logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +# REQUEST_TIMEOUT = 5000 +REQUEST_RETRIES = 3 + +pid = os.getpid() + +class ClientContext: + def __init__(self, server_ip, server_port, identity): + self.identity = identity + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.connect((server_ip, server_port)) + + def submit_dataloaderinfo(self, is_train, dataloader_info): + send_msg(self.socket, (DATALOADER_REQ, is_train, self.identity, dataloader_info)) + + def submit_epoch_req(self, is_train, epoch_idx): + send_msg(self.socket, (EPOCH_REQ, is_train, epoch_idx)) + + def stop_epoch(self, is_train, batch_idx): + send_msg(self.socket, (EPOCH_STOP, is_train, batch_idx)) + + def recv_dataset_metadata(self): + return recv_msg(self.socket) + + +client = None + +class FlexGVirtDataLoader(): + def __init__(self, train=True, # train or test + remote_dataset_path=None, regular_dataset=None, # remote dataset filepath or PyTorch built-in datasets + batch_size=1, # number of samples per batch + shuffle=False, # whether to shuffle the data at every epoch + sampler=None, # defines the strategy to draw samples from the dataset + batch_sampler=None, # defines the strategy to draw batches from the dataset (SequentialSampler, RandomSampler, WeightedSampler, etc.) + collate_fn=None, # merges a list of samples to form a mini-batch of Tensor(s) (e.g. image, label pairs) + drop_last=False, # whether to drop the last incomplete batch, if the dataset size is not divisible by the batch size + preprocess=None, # preprocessing function for each sample + generator=None, # random number generator + num_workers=0, # number of subprocesses to use for dataloading + other_dataloader=None, # other dataloader + proxy=True, # whether to use GPU proxy & Resource Dispatcher + distributed=False, # whether to use distributed data parallelism + seed=0, # random seed used to shuffle the sampler, identical across all processes in the distributed group + rank=None, # rank of the current process in the distributed group + world_size=None, # number of processes in the distributed group + sample_drop_last=False, # If ``False``, the sampler will add extra indices to make the data evenly divisible across the replicas. + ): + torch.cuda.mem_get_info() # todo + clientId = get_client_id() + identity = str(clientId) + "_" + str(pid) + logging.debug(f"dataloader identity: {identity}") + self.batch_num = 0 + self.train = train + + batch_info_bytes = struct.calcsize('bN') #! be careful with the alignment + self.shm_obj = posix_ipc.SharedMemory(name=("/flexgv_shm_%s_datatype" % identity), flags=posix_ipc.O_CREAT|posix_ipc.O_RDWR, size=(batch_info_bytes + 4 + 40)) # clientID_PID_datatype (contains: struct batch info + server ip and port) + self.shm_desc = mmap.mmap(self.shm_obj.fd, self.shm_obj.size) + self.shm_write_byte = (1 if self.train else 2).to_bytes(1, byteorder='little') # 1 for train, 2 for test/validation, 3 for model + self.shm_desc.seek(0) + self.shm_desc.write((0).to_bytes(1, byteorder='little')) # init the data type + + global client + if client is None: + if proxy: + self.shm_desc.seek(batch_info_bytes) + server_ip_port = self.shm_desc.read(4 + 40) # read server ip and port + server_port, server_ip_bytes = struct.unpack('i40s', server_ip_port) + server_ip = server_ip_bytes.decode('utf-8').strip('\x00') + else: # todo: to be validated + server_ip, server_port = get_server_ep(need_clientId = True) + client = ClientContext(server_ip, server_port, identity) + + self.remote_dataset_path = remote_dataset_path + self.dataset = regular_dataset + self.batch_size = batch_size + self.shuffle = shuffle + self.sampler = sampler + self.batch_sampler = batch_sampler + self.collate_fn = collate_fn + self.drop_last = drop_last + self.preprocess = preprocess + self.generator = generator + self.num_workers = num_workers + + self.distributed = distributed + self.seed = seed + self.epoch = 0 + if self.distributed: + if world_size is None: + if not dist.is_available(): + raise RuntimeError("Requires distributed package to be available") + world_size = dist.get_world_size() + if rank is None: + if not dist.is_available(): + raise RuntimeError("Requires distributed package to be available") + rank = dist.get_rank() + if rank >= world_size or rank < 0: + raise ValueError( + "Invalid rank {}, rank should be in the interval" + " [0, {}]".format(rank, world_size - 1)) + + self.rank = rank + self.world_size = world_size + logging.debug(f"rank={self.rank}, world_size={self.world_size}") + + logging.debug("MyDataLoader __init__ called") + client.submit_dataloaderinfo(self.train, DataloaderInfo(train, remote_dataset_path, regular_dataset, + batch_size, shuffle, sampler, batch_sampler, + collate_fn, drop_last, preprocess, + generator, num_workers, other_dataloader, + distributed, seed, rank, world_size, sample_drop_last)) + # self.batch_info = client.recv_batch_info() + self.dataset_metadata = client.recv_dataset_metadata() + self.batch_num = self.dataset_metadata.batch_num + self.stop_batch_idx = self.batch_num + self.set_epoch(0) + + logging.debug(f"Received dataset metadata(batch info): {self.dataset_metadata}") + + def set_epoch(self, epoch): + self.epoch = epoch + logging.debug(f"set_epoch called, epoch={self.epoch}") + + def stop(self): + self.stop_batch_idx = self.batch_idx # store the current batch index + self.batch_idx = self.batch_num # to stop the iteration + + def __iter__(self): + logging.debug(f"iter__init__ called, is_train={self.train}") + self.batch_idx = 0 + self.shm_desc.seek(0) + self.shm_desc.write(self.shm_write_byte) # notify the CUDA wrapper to start the data loading process + global client + client.submit_epoch_req(self.train, self.epoch) # notify the server to start the data loading process + # self.batch_info = client.recv_batch_info() # recv batch info again just for blocking the process + self.dataset_metadata = client.recv_dataset_metadata() # recv dataset metadata again just for blocking the process + self.batch_num = self.dataset_metadata.batch_num + self.stop_batch_idx = self.batch_num + + self.first_data = () if self.dataset_metadata.is_iterable else {} + self.first_batch_size = 0 + for col in self.dataset_metadata.first_batch: + data = torch.zeros(col["shape"], dtype=col["dtype"]) + self.first_batch_size += data.numel() * data.element_size() + if self.dataset_metadata.is_iterable: + self.first_data += (data,) + else: + self.first_data[col["key"]] = data + + self.last_data = () if self.dataset_metadata.is_iterable else {} + self.last_batch_size = 0 + for col in self.dataset_metadata.last_batch: + data = torch.zeros(col["shape"], dtype=col["dtype"]) + self.last_batch_size += data.numel() * data.element_size() + if self.dataset_metadata.is_iterable: + self.last_data += (data,) + else: + self.last_data[col["key"]] = data + + logging.debug(f"first_batch_size: {self.first_batch_size}, last_batch_size: {self.last_batch_size}") + + return self + + def __next__(self): + logging.debug(f"iter__next__ called, batch_idx={self.batch_idx}, batch_num={self.batch_num}") + if self.batch_idx < self.batch_num - 1: + self.batch_idx += 1 + self.shm_desc.seek(0) + self.shm_desc.write(struct.pack('bN', (1 if self.train else 2), self.first_batch_size)) + return self.first_data + elif self.batch_idx == self.batch_num - 1: + self.batch_idx += 1 + self.shm_desc.seek(0) + self.shm_desc.write(struct.pack('bN', (1 if self.train else 2), self.last_batch_size)) + return self.last_data + else: + self.shm_desc.seek(0) + self.shm_desc.write((0).to_bytes(1, byteorder='little')) # clear the data type + global client + client.stop_epoch(self.train, self.stop_batch_idx) # batch_idx indicates the next batch to be loaded + raise StopIteration + + def __len__(self): + return self.batch_num + + def __del__(self): + self.shm_desc.close() + self.shm_obj.close_fd() + + diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/preprocess_utils.py b/GPU-Virtual-Service/gpu-remoting/src/storage/preprocess_utils.py new file mode 100644 index 0000000..cc00643 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/preprocess_utils.py @@ -0,0 +1,134 @@ +import random +import numpy as np +import scipy.ndimage +from torch.utils.data import Dataset +from torchvision import transforms + + +def get_train_transforms(): + rand_flip = RandFlip() + cast = Cast(types=(np.float32, np.uint8)) + rand_scale = RandomBrightnessAugmentation(factor=0.3, prob=0.1) + rand_noise = GaussianNoise(mean=0.0, std=0.1, prob=0.1) + rand_crop = RandBalancedCrop(patch_size=[128, 128, 128], oversampling=0.4) + train_transforms = transforms.Compose([rand_crop, rand_flip, cast, rand_scale, rand_noise]) + return train_transforms + +class RandBalancedCrop: + def __init__(self, patch_size, oversampling): + self.patch_size = patch_size + self.oversampling = oversampling + + def __call__(self, data): + image, label = data["image"], data["label"] + if random.random() < self.oversampling: + image, label, cords = self.rand_foreg_cropd(image, label) + else: + image, label, cords = self._rand_crop(image, label) + data.update({"image": image, "label": label}) + return data + + @staticmethod + def randrange(max_range): + return 0 if max_range == 0 else random.randrange(max_range) + + def get_cords(self, cord, idx): + return cord[idx], cord[idx] + self.patch_size[idx] + + def _rand_crop(self, image, label): + ranges = [s - p for s, p in zip(image.shape[1:], self.patch_size)] + cord = [self.randrange(x) for x in ranges] + low_x, high_x = self.get_cords(cord, 0) + low_y, high_y = self.get_cords(cord, 1) + low_z, high_z = self.get_cords(cord, 2) + image = image[:, low_x:high_x, low_y:high_y, low_z:high_z] + label = label[:, low_x:high_x, low_y:high_y, low_z:high_z] + return image, label, [low_x, high_x, low_y, high_y, low_z, high_z] + + def rand_foreg_cropd(self, image, label): + def adjust(foreg_slice, patch_size, label, idx): + diff = patch_size[idx - 1] - (foreg_slice[idx].stop - foreg_slice[idx].start) + sign = -1 if diff < 0 else 1 + diff = abs(diff) + ladj = self.randrange(diff) + hadj = diff - ladj + low = max(0, foreg_slice[idx].start - sign * ladj) + high = min(label.shape[idx], foreg_slice[idx].stop + sign * hadj) + diff = patch_size[idx - 1] - (high - low) + if diff > 0 and low == 0: + high += diff + elif diff > 0: + low -= diff + return low, high + + cl = np.random.choice(np.unique(label[label > 0])) + foreg_slices = scipy.ndimage.find_objects(scipy.ndimage.measurements.label(label==cl)[0]) + foreg_slices = [x for x in foreg_slices if x is not None] + slice_volumes = [np.prod([s.stop - s.start for s in sl]) for sl in foreg_slices] + slice_idx = np.argsort(slice_volumes)[-2:] + foreg_slices = [foreg_slices[i] for i in slice_idx] + if not foreg_slices: + return self._rand_crop(image, label) + foreg_slice = foreg_slices[random.randrange(len(foreg_slices))] + low_x, high_x = adjust(foreg_slice, self.patch_size, label, 1) + low_y, high_y = adjust(foreg_slice, self.patch_size, label, 2) + low_z, high_z = adjust(foreg_slice, self.patch_size, label, 3) + image = image[:, low_x:high_x, low_y:high_y, low_z:high_z] + label = label[:, low_x:high_x, low_y:high_y, low_z:high_z] + return image, label, [low_x, high_x, low_y, high_y, low_z, high_z] + + +class RandFlip: + def __init__(self): + self.axis = [1, 2, 3] + self.prob = 1 / len(self.axis) + + def flip(self, data, axis): + data["image"] = np.flip(data["image"], axis=axis).copy() + data["label"] = np.flip(data["label"], axis=axis).copy() + return data + + def __call__(self, data): + for axis in self.axis: + if random.random() < self.prob: + data = self.flip(data, axis) + return data + + +class Cast: + def __init__(self, types): + self.types = types + + def __call__(self, data): + data["image"] = data["image"].astype(self.types[0]) + data["label"] = data["label"].astype(self.types[1]) + return data + + +class RandomBrightnessAugmentation: + def __init__(self, factor, prob): + self.prob = prob + self.factor = factor + + def __call__(self, data): + image = data["image"] + if random.random() < self.prob: + factor = np.random.uniform(low=1.0-self.factor, high=1.0+self.factor, size=1) + image = (image * (1 + factor)).astype(image.dtype) + data.update({"image": image}) + return data + + +class GaussianNoise: + def __init__(self, mean, std, prob): + self.mean = mean + self.std = std + self.prob = prob + + def __call__(self, data): + image = data["image"] + if random.random() < self.prob: + scale = np.random.uniform(low=0.0, high=self.std) + noise = np.random.normal(loc=self.mean, scale=scale, size=image.shape).astype(image.dtype) + data.update({"image": image + noise}) + return data \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/shareMemoryPy.bak b/GPU-Virtual-Service/gpu-remoting/src/storage/shareMemoryPy.bak new file mode 100644 index 0000000..c90ad79 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/shareMemoryPy.bak @@ -0,0 +1,89 @@ +import traceback +import numpy as np +import mmap +import os +import posix_ipc +import pdb + +shared_memory_size = 4096 * 4096 * 4096 +shared_memory_model = "shm_model" +shared_memory_data = "shm_data" +import mmap +import numpy as np +import posix_ipc +import multiprocessing + + +class SharedMemoryObj: + def __init__(self, name, size): + try: + self.shm = posix_ipc.SharedMemory(name=name, flags=posix_ipc.O_CREAT, size=size) + except posix_ipc.ExistentialError: + self.shm = posix_ipc.SharedMemory(name=name) + + # 将共享内存映射到当前进程的地址空间 + self.mapped_memory = mmap.mmap(self.shm.fd, self.shm.size) + # 创建锁 + self.lock = multiprocessing.Lock() + + def overwrite(self, batch): + if not isinstance(batch, np.ndarray): + raise ValueError("Input must be a NumPy array") + + with self.lock: + self.mapped_memory.seek(0) + bytes_batch = batch.tobytes() + self.mapped_memory.write(bytes_batch) + self.mapped_memory.flush() + print("Data overwritten to shared memory.") + + def append(self, batch): + if not isinstance(batch, np.ndarray): + raise ValueError("Input must be a NumPy array") + + with self.lock: + # 获取当前文件指针位置 + current_position = self.mapped_memory.tell() + # 将文件指针移到当前位置 + self.mapped_memory.seek(current_position) + bytes_batch = batch.tobytes() + self.mapped_memory.write(bytes_batch) + self.mapped_memory.flush() + print("Data appended to shared memory.") + + def read(self, size, offset): + with self.lock: + self.mapped_memory.seek(offset) + return self.mapped_memory.read(size) + + def close(self): + """ + Unlink and close the shared memory. + """ + try: + # Unlink the shared memory + self.shm.unlink() + # Close the mapped memory + self.mapped_memory.close() + except Exception as e: + print(f"Error: {e}") + + + +import numpy as np +import time + +def write_data_to_shared_memory(): + shared_memory = SharedMemoryObj(name='/example_shared_memory', size=1024) + + try: + for i in range(5): + data_to_write = np.array([i, i + 1, i + 2], dtype=np.int32) + shared_memory.append(data_to_write) + time.sleep(2) # Simulate other operations + + finally: + shared_memory.close() + +# if __name__ == "__main__": +# write_data_to_shared_memory() \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/storageEndpoint.py b/GPU-Virtual-Service/gpu-remoting/src/storage/storageEndpoint.py new file mode 100644 index 0000000..75a0a65 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/storageEndpoint.py @@ -0,0 +1,89 @@ +import socket +import errno +# import multiprocessing as mp +import torch.multiprocessing as mp +from dataFeeder import DataFeeder +from common import DataloaderInfo, BatchInfo, get_server_ep, logging, send_msg, recv_msg, EPOCH_REQ, EPOCH_STOP, DATALOADER_REQ + +def handle_client(client_socket, client_addr, rank): + logging.info(f"Accepted connection from {client_addr}") + datafeeders = [None, None] + client_ident = None + + try: + while True: + infoObj = recv_msg(client_socket) + if infoObj is None: + break + + request_type = infoObj[0] + is_train = infoObj[1] + if request_type == DATALOADER_REQ: + if client_ident is None: + client_ident = infoObj[2] + dataloader_info = infoObj[3] + logging.debug(f"[{client_ident}] new dataloader info: {dataloader_info}") + + datafeeder = DataFeeder(client_ident, rank, dataloader_info) + datafeeders[is_train] = datafeeder + # create the shared memory queue + send_msg(client_socket, datafeeder.get_batch_info(is_first=True)) + + elif request_type == EPOCH_REQ: + client_epoch_idx = infoObj[2] + logging.info(f"[{client_ident}] new epoch request (train={is_train}, epoch={client_epoch_idx})") + datafeeders[is_train].reset(client_epoch_idx) + send_msg(client_socket, datafeeders[is_train].get_batch_info(is_first=False)) + datafeeders[is_train].start_load_data() + elif request_type == EPOCH_STOP: + client_batch_idx = infoObj[2] + logging.info(f"[{client_ident}] epoch stop request (train={is_train}, client_batch_idx={client_batch_idx})") + datafeeders[is_train].stop_load_data(client_batch_idx) + else: + logging.error(f"Received unknown message: {infoObj}") + except socket.error as e: + if e.errno == errno.ECONNRESET: + pass + else: + logging.error(f"Unexpected socket error: {e}") + except Exception as e: + logging.error(f"Exception: {e}") + finally: + for datafeeder in datafeeders: + if datafeeder is not None: + datafeeder.stop_load_data() + logging.info(f"Connection from client#{client_ident}{client_addr} closed") + client_socket.close() + + + +def listener(server_address): + conn_num = 0 + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + server_socket.bind(server_address) + except OSError as e: + if e.errno == errno.EADDRINUSE: + logging.error(f"Address {server_address[0]}:{server_address[1]} is already in use") + return + else: + logging.error(f"Unexpected error: {e}") + return + server_socket.listen() + logging.info(f"Listening on {server_address[0]}:{server_address[1]}") + try: + while True: + client_socket, client_addr = server_socket.accept() + conn_num += 1 + process = mp.Process(target=handle_client, args=(client_socket, client_addr, conn_num)) + process.start() + client_socket.close() # close the socket in parent process, let the child process handle dataloading + except KeyboardInterrupt: + logging.info("Shutting down...") + except Exception as e: + logging.error(f"Error: {e}") + finally: + server_socket.close() + +if __name__ == "__main__": + listener(get_server_ep()) \ No newline at end of file