From b93596567ca2c06e946cc4136599a8894572aa62 Mon Sep 17 00:00:00 2001 From: madd_2014 Date: Mon, 6 Jul 2026 22:21:46 +0800 Subject: [PATCH] feat(gpu-remoting): add server-side CUDA callback handlers, storage data pipeline, and validation test samples Add three core modules for the GPU virtualization remoting service: - server: Implement RPC-based server endpoint with ~150 CUDA/cuBLAS/cuDNN/NCCL API callback handlers, virtual-to-real address mapping, checkpoint backup/recovery, and multi-client scheduling support - storage: Build the remote data feeding pipeline with shared-memory ring queue for zero-copy batch transfer to GPU, PyTorch DataLoader-compatible transparent API (FlexGVirtDataLoader), and 3D medical imaging preprocessing utilities - sample: Add comprehensive CUDA validation test suite covering memory management, kernel launch, stream/event/device APIs, CUDA Graph, multi-threading, cuBLAS, cuBLASLt, cuDNN, NCCL multi-node communication, and performance benchmarks --- .../gpu-remoting/src/sample/BlackScholes.cu | 265 ++++ .../gpu-remoting/src/sample/alignedTypes.cu | 336 +++++ .../gpu-remoting/src/sample/asyncAPI.cu | 142 ++ .../gpu-remoting/src/sample/bandwidthTest.cu | 969 ++++++++++++++ .../gpu-remoting/src/sample/clock.cu | 178 +++ .../src/sample/convolutionSeparable.cu | 182 +++ .../src/sample/fastWalshTransform.cu | 242 ++++ .../sample/include/BlackScholes_kernel.cuh | 167 +++ .../include/convolutionSeparable_common.cuh | 268 ++++ .../src/sample/include/exception.h | 151 +++ .../include/fastWalshTransform_kernel.cuh | 185 +++ .../src/sample/include/helper_cuda.h | 976 ++++++++++++++ .../src/sample/include/helper_functions.h | 59 + .../src/sample/include/helper_image.h | 1001 ++++++++++++++ .../src/sample/include/helper_string.h | 428 ++++++ .../src/sample/include/helper_timer.h | 465 +++++++ .../src/sample/include/nccl_util.h | 66 + .../gpu-remoting/src/sample/testCuDNN.cu | 339 +++++ .../src/sample/testCuDNNBackend.cu.bak | 184 +++ .../gpu-remoting/src/sample/testCuDNNTrans.cu | 51 + .../gpu-remoting/src/sample/testCublas.cu | 63 + .../gpu-remoting/src/sample/testCublasLt.cu | 125 ++ .../gpu-remoting/src/sample/testDevice.cu | 165 +++ .../gpu-remoting/src/sample/testEvent.cu | 24 + .../gpu-remoting/src/sample/testGraph.cu | 79 ++ .../gpu-remoting/src/sample/testHighThread.cu | 117 ++ .../src/sample/testKernelLaunch.cu | 87 ++ .../src/sample/testKernelwithStruct.cu | 82 ++ .../gpu-remoting/src/sample/testMemory.cu | 82 ++ .../gpu-remoting/src/sample/testMemory2.cu | 41 + .../gpu-remoting/src/sample/testNcclClient.cu | 198 +++ .../gpu-remoting/src/sample/testNcclLocal.cu | 147 ++ .../gpu-remoting/src/sample/testNcclServer.cu | 216 +++ .../gpu-remoting/src/sample/testOthers.cu | 32 + .../gpu-remoting/src/sample/testStream.cu | 68 + .../gpu-remoting/src/sample/testT.cu | 88 ++ .../gpu-remoting/src/sample/testThread.cu | 102 ++ .../gpu-remoting/src/sample/testZeroCopy.cu | 87 ++ .../gpu-remoting/src/server/cuBlasCallback.cc | 185 +++ .../src/server/cuBlasLtCallback.cc | 316 +++++ .../gpu-remoting/src/server/cuDNNCallback.cc | 1179 +++++++++++++++++ .../src/server/cudaRtDeviceCallback.cc | 155 +++ .../src/server/cudaRtEventCallback.cc | 161 +++ .../src/server/cudaRtExecutionCallback.cc | 250 ++++ .../src/server/cudaRtInternalCallback.cc | 319 +++++ .../src/server/cudaRtMemoryCallback.cc | 467 +++++++ .../src/server/cudaRtOtherCallback.cc | 80 ++ .../src/server/cudaRtStreamCallback.cc | 312 +++++ .../gpu-remoting/src/server/ncclCallback.cc | 689 ++++++++++ .../gpu-remoting/src/server/serverApp.cc | 103 ++ .../gpu-remoting/src/server/serverBackup.cc | 657 +++++++++ .../gpu-remoting/src/server/serverEndpoint.cc | 513 +++++++ .../gpu-remoting/src/storage/Design.png | Bin 0 -> 167076 bytes .../gpu-remoting/src/storage/PyShmqueue.py | 206 +++ .../src/storage/code_overview.png | Bin 0 -> 90623 bytes .../gpu-remoting/src/storage/common.py | 104 ++ .../gpu-remoting/src/storage/dataFeeder.py | 258 ++++ .../src/storage/dataloaderWrapper.py | 220 +++ .../src/storage/preprocess_utils.py | 134 ++ .../src/storage/shareMemoryPy.bak | 89 ++ .../src/storage/storageEndpoint.py | 89 ++ 61 files changed, 14943 insertions(+) create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/BlackScholes.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/alignedTypes.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/asyncAPI.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/bandwidthTest.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/clock.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/convolutionSeparable.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/fastWalshTransform.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/include/BlackScholes_kernel.cuh create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/include/convolutionSeparable_common.cuh create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/include/exception.h create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/include/fastWalshTransform_kernel.cuh create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_cuda.h create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_functions.h create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_image.h create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_string.h create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_timer.h create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/include/nccl_util.h create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNN.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNBackend.cu.bak create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNTrans.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testCublas.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testCublasLt.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testDevice.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testEvent.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testGraph.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testHighThread.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testKernelLaunch.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testKernelwithStruct.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testMemory.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testMemory2.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testNcclClient.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testNcclLocal.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testNcclServer.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testOthers.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testStream.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testT.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testThread.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/sample/testZeroCopy.cu create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/cuBlasCallback.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/cuBlasLtCallback.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/cuDNNCallback.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/cudaRtDeviceCallback.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/cudaRtEventCallback.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/cudaRtExecutionCallback.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/cudaRtInternalCallback.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/cudaRtMemoryCallback.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/cudaRtOtherCallback.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/cudaRtStreamCallback.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/ncclCallback.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/serverApp.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/serverBackup.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/server/serverEndpoint.cc create mode 100644 GPU-Virtual-Service/gpu-remoting/src/storage/Design.png create mode 100644 GPU-Virtual-Service/gpu-remoting/src/storage/PyShmqueue.py create mode 100644 GPU-Virtual-Service/gpu-remoting/src/storage/code_overview.png create mode 100644 GPU-Virtual-Service/gpu-remoting/src/storage/common.py create mode 100644 GPU-Virtual-Service/gpu-remoting/src/storage/dataFeeder.py create mode 100644 GPU-Virtual-Service/gpu-remoting/src/storage/dataloaderWrapper.py create mode 100644 GPU-Virtual-Service/gpu-remoting/src/storage/preprocess_utils.py create mode 100644 GPU-Virtual-Service/gpu-remoting/src/storage/shareMemoryPy.bak create mode 100644 GPU-Virtual-Service/gpu-remoting/src/storage/storageEndpoint.py diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/BlackScholes.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/BlackScholes.cu new file mode 100644 index 0000000..1512f1a --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/BlackScholes.cu @@ -0,0 +1,265 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This sample evaluates fair call and put prices for a + * given set of European options by Black-Scholes formula. + * See supplied whitepaper for more explanations. + */ + +#include // helper functions for string parsing +#include // helper functions CUDA error checking and initialization + +//////////////////////////////////////////////////////////////////////////////// +// Process an array of optN options on CPU +//////////////////////////////////////////////////////////////////////////////// +extern "C" void BlackScholesCPU(float *h_CallResult, float *h_PutResult, + float *h_StockPrice, float *h_OptionStrike, + float *h_OptionYears, float Riskfree, + float Volatility, int optN); + +//////////////////////////////////////////////////////////////////////////////// +// Process an array of OptN options on GPU +//////////////////////////////////////////////////////////////////////////////// +#include "./include/BlackScholes_kernel.cuh" + +//////////////////////////////////////////////////////////////////////////////// +// Helper function, returning uniformly distributed +// random float in [low, high] range +//////////////////////////////////////////////////////////////////////////////// +float RandFloat(float low, float high) { + float t = (float)rand() / (float)RAND_MAX; + return (1.0f - t) * low + t * high; +} + +//////////////////////////////////////////////////////////////////////////////// +// Data configuration +//////////////////////////////////////////////////////////////////////////////// +const int OPT_N = 4000000; +const int NUM_ITERATIONS = 512; + +const int OPT_SZ = OPT_N * sizeof(float); +const float RISKFREE = 0.02f; +const float VOLATILITY = 0.30f; + +#define DIV_UP(a, b) (((a) + (b)-1) / (b)) + +StopWatchInterface *totalTimer = NULL; + +//////////////////////////////////////////////////////////////////////////////// +// Main program +//////////////////////////////////////////////////////////////////////////////// +int main(int argc, char **argv) { + char* logDir = NULL; + if (checkCmdLineFlag(argc, (const char **)argv, "logDir")) { + getCmdLineArgumentString(argc, (const char **)argv, "logDir=", &logDir); + } + + sdkCreateTimer(&totalTimer); + sdkStartTimer(&totalTimer); + + // Start logs + printf("[%s] - Starting...\n", argv[0]); + + //'h_' prefix - CPU (host) memory space + float + // Results calculated by CPU for reference + *h_CallResultCPU, + *h_PutResultCPU, + // CPU copy of GPU results + *h_CallResultGPU, *h_PutResultGPU, + // CPU instance of input data + *h_StockPrice, *h_OptionStrike, *h_OptionYears; + + //'d_' prefix - GPU (device) memory space + float + // Results calculated by GPU + *d_CallResult, + *d_PutResult, + // GPU instance of input data + *d_StockPrice, *d_OptionStrike, *d_OptionYears; + + double delta, ref, sum_delta, sum_ref, max_delta, L1norm, gpuTime; + + StopWatchInterface *hTimer = NULL; + int i; + + findCudaDevice(argc, (const char **)argv); + + sdkCreateTimer(&hTimer); + + printf("Initializing data...\n"); + printf("...allocating CPU memory for options.\n"); + h_CallResultCPU = (float *)malloc(OPT_SZ); + h_PutResultCPU = (float *)malloc(OPT_SZ); + h_CallResultGPU = (float *)malloc(OPT_SZ); + h_PutResultGPU = (float *)malloc(OPT_SZ); + h_StockPrice = (float *)malloc(OPT_SZ); + h_OptionStrike = (float *)malloc(OPT_SZ); + h_OptionYears = (float *)malloc(OPT_SZ); + + printf("...allocating GPU memory for options.\n"); + checkCudaErrors(cudaMalloc((void **)&d_CallResult, OPT_SZ)); + checkCudaErrors(cudaMalloc((void **)&d_PutResult, OPT_SZ)); + checkCudaErrors(cudaMalloc((void **)&d_StockPrice, OPT_SZ)); + checkCudaErrors(cudaMalloc((void **)&d_OptionStrike, OPT_SZ)); + checkCudaErrors(cudaMalloc((void **)&d_OptionYears, OPT_SZ)); + + printf("...generating input data in CPU mem.\n"); + srand(5347); + + // Generate options set + for (i = 0; i < OPT_N; i++) { + h_CallResultCPU[i] = 0.0f; + h_PutResultCPU[i] = -1.0f; + h_StockPrice[i] = RandFloat(5.0f, 30.0f); + h_OptionStrike[i] = RandFloat(1.0f, 100.0f); + h_OptionYears[i] = RandFloat(0.25f, 10.0f); + } + + printf("...copying input data to GPU mem.\n"); + // Copy options data to GPU memory for further processing + checkCudaErrors( + cudaMemcpy(d_StockPrice, h_StockPrice, OPT_SZ, cudaMemcpyHostToDevice)); + checkCudaErrors(cudaMemcpy(d_OptionStrike, h_OptionStrike, OPT_SZ, + cudaMemcpyHostToDevice)); + checkCudaErrors( + cudaMemcpy(d_OptionYears, h_OptionYears, OPT_SZ, cudaMemcpyHostToDevice)); + printf("Data init done.\n\n"); + + printf("Executing Black-Scholes GPU kernel (%i iterations)...\n", + NUM_ITERATIONS); + checkCudaErrors(cudaDeviceSynchronize()); + sdkResetTimer(&hTimer); + sdkStartTimer(&hTimer); + + for (i = 0; i < NUM_ITERATIONS; i++) { + BlackScholesGPU<<>>( + (float2 *)d_CallResult, (float2 *)d_PutResult, (float2 *)d_StockPrice, + (float2 *)d_OptionStrike, (float2 *)d_OptionYears, RISKFREE, VOLATILITY, + OPT_N); + getLastCudaError("BlackScholesGPU() execution failed\n"); + } + + checkCudaErrors(cudaDeviceSynchronize()); + sdkStopTimer(&hTimer); + gpuTime = sdkGetTimerValue(&hTimer) / NUM_ITERATIONS; + + // Both call and put is calculated + printf("Options count : %i \n", 2 * OPT_N); + printf("BlackScholesGPU() time : %f msec\n", gpuTime); + printf("Effective memory bandwidth: %f GB/s\n", + ((double)(5 * OPT_N * sizeof(float)) * 1E-9) / (gpuTime * 1E-3)); + printf("Gigaoptions per second : %f \n\n", + ((double)(2 * OPT_N) * 1E-9) / (gpuTime * 1E-3)); + + printf( + "BlackScholes, Throughput = %.4f GOptions/s, Time = %.5f s, Size = %u " + "options, NumDevsUsed = %u, Workgroup = %u\n", + (((double)(2.0 * OPT_N) * 1.0E-9) / (gpuTime * 1.0E-3)), gpuTime * 1e-3, + (2 * OPT_N), 1, 128); + + printf("\nReading back GPU results...\n"); + // Read back GPU results to compare them to CPU results + checkCudaErrors(cudaMemcpy(h_CallResultGPU, d_CallResult, OPT_SZ, + cudaMemcpyDeviceToHost)); + checkCudaErrors( + cudaMemcpy(h_PutResultGPU, d_PutResult, OPT_SZ, cudaMemcpyDeviceToHost)); + + printf("Checking the results...\n"); + printf("...running CPU calculations.\n\n"); + // Calculate options values on CPU + BlackScholesCPU(h_CallResultCPU, h_PutResultCPU, h_StockPrice, h_OptionStrike, + h_OptionYears, RISKFREE, VOLATILITY, OPT_N); + + printf("Comparing the results...\n"); + // Calculate max absolute difference and L1 distance + // between CPU and GPU results + sum_delta = 0; + sum_ref = 0; + max_delta = 0; + + for (i = 0; i < OPT_N; i++) { + ref = h_CallResultCPU[i]; + delta = fabs(h_CallResultCPU[i] - h_CallResultGPU[i]); + + if (delta > max_delta) { + max_delta = delta; + } + + sum_delta += delta; + sum_ref += fabs(ref); + } + + L1norm = sum_delta / sum_ref; + printf("L1 norm: %E\n", L1norm); + printf("Max absolute error: %E\n\n", max_delta); + + printf("Shutting down...\n"); + printf("...releasing GPU memory.\n"); + checkCudaErrors(cudaFree(d_OptionYears)); + checkCudaErrors(cudaFree(d_OptionStrike)); + checkCudaErrors(cudaFree(d_StockPrice)); + checkCudaErrors(cudaFree(d_PutResult)); + checkCudaErrors(cudaFree(d_CallResult)); + + printf("...releasing CPU memory.\n"); + free(h_OptionYears); + free(h_OptionStrike); + free(h_StockPrice); + free(h_PutResultGPU); + free(h_CallResultGPU); + free(h_PutResultCPU); + free(h_CallResultCPU); + sdkDeleteTimer(&hTimer); + printf("Shutdown done.\n"); + + printf("\n[BlackScholes] - Test Summary\n"); + + if (L1norm > 1e-6) { + printf("Test failed!\n"); + exit(EXIT_FAILURE); + } + + printf( + "\nNOTE: The CUDA Samples are not meant for performance measurements. " + "Results may vary when GPU Boost is enabled.\n\n"); + printf("Test passed\n"); + + sdkStopTimer(&totalTimer); + double totalTime = sdkGetTimerValue(&totalTimer); + printf("Total time: %f ms \n", totalTime); + if (logDir != NULL) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << totalTime << std::endl; + outFile.close(); + } + + sdkDeleteTimer(&totalTimer); + + exit(EXIT_SUCCESS); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/alignedTypes.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/alignedTypes.cu new file mode 100644 index 0000000..1d91b01 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/alignedTypes.cu @@ -0,0 +1,336 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This is a simple test showing huge access speed gap + * between aligned and misaligned structures + * (those having/missing __align__ keyword). + * It measures per-element copy throughput for + * aligned and misaligned structures on + * big chunks of data. + */ + +// includes, system +#include +#include +#include +#include +#include + +// includes, project +#include // helper functions for CUDA error checking and initialization +#include // helper utility functions + +//////////////////////////////////////////////////////////////////////////////// +// Misaligned types +//////////////////////////////////////////////////////////////////////////////// +typedef unsigned char uint8; + +typedef unsigned short int uint16; + +typedef struct { + unsigned char r, g, b, a; +} RGBA8_misaligned; + +typedef struct { + unsigned int l, a; +} LA32_misaligned; + +typedef struct { + unsigned int r, g, b; +} RGB32_misaligned; + +typedef struct { + unsigned int r, g, b, a; +} RGBA32_misaligned; + +//////////////////////////////////////////////////////////////////////////////// +// Aligned types +//////////////////////////////////////////////////////////////////////////////// +typedef struct __align__(4) { + unsigned char r, g, b, a; +} +RGBA8; + +typedef unsigned int I32; + +typedef struct __align__(8) { + unsigned int l, a; +} +LA32; + +typedef struct __align__(16) { + unsigned int r, g, b; +} +RGB32; + +typedef struct __align__(16) { + unsigned int r, g, b, a; +} +RGBA32; + +//////////////////////////////////////////////////////////////////////////////// +// Because G80 class hardware natively supports global memory operations +// only with data elements of 4, 8 and 16 bytes, if structure size +// exceeds 16 bytes, it can't be efficiently read or written, +// since more than one global memory non-coalescable load/store instructions +// will be generated, even if __align__ option is supplied. +// "Structure of arrays" storage strategy offers best performance +// in general case. See section 5.1.2 of the Programming Guide. +//////////////////////////////////////////////////////////////////////////////// +typedef struct __align__(16) { + RGBA32 c1, c2; +} +RGBA32_2; + +//////////////////////////////////////////////////////////////////////////////// +// Common host and device functions +//////////////////////////////////////////////////////////////////////////////// +// Round a / b to nearest higher integer value +int iDivUp(int a, int b) { return (a % b != 0) ? (a / b + 1) : (a / b); } + +// Round a / b to nearest lower integer value +int iDivDown(int a, int b) { return a / b; } + +// Align a to nearest higher multiple of b +int iAlignUp(int a, int b) { return (a % b != 0) ? (a - a % b + b) : a; } + +// Align a to nearest lower multiple of b +int iAlignDown(int a, int b) { return a - a % b; } + +//////////////////////////////////////////////////////////////////////////////// +// Simple CUDA kernel. +// Copy is carried out on per-element basis, +// so it's not per-byte in case of padded structures. +//////////////////////////////////////////////////////////////////////////////// +template +__global__ void testKernel(TData *d_odata, TData *d_idata, int numElements) { + const int tid = blockDim.x * blockIdx.x + threadIdx.x; + const int numThreads = blockDim.x * gridDim.x; + + for (int pos = tid; pos < numElements; pos += numThreads) { + d_odata[pos] = d_idata[pos]; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Validation routine for simple copy kernel. +// We must know "packed" size of TData (number_of_fields * sizeof(simple_type)) +// and compare only these "packed" parts of the structure, +// containing actual user data. The compiler behavior with padding bytes +// is undefined, since padding is merely a placeholder +// and doesn't contain any user data. +//////////////////////////////////////////////////////////////////////////////// +template +int testCPU(TData *h_odata, TData *h_idata, int numElements, + int packedElementSize) { + for (int pos = 0; pos < numElements; pos++) { + TData src = h_idata[pos]; + TData dst = h_odata[pos]; + + for (int i = 0; i < packedElementSize; i++) + if (((char *)&src)[i] != ((char *)&dst)[i]) { + return 0; + } + } + + return 1; +} + +//////////////////////////////////////////////////////////////////////////////// +// Data configuration +//////////////////////////////////////////////////////////////////////////////// +// Memory chunk size in bytes. Reused for test +const int MEM_SIZE = 50000000; +const int NUM_ITERATIONS = 32; + +// GPU input and output data +unsigned char *d_idata, *d_odata; +// CPU input data and instance of GPU output data +unsigned char *h_idataCPU, *h_odataGPU; +StopWatchInterface *hTimer = NULL; +StopWatchInterface *totalTimer = NULL; + +template +int runTest(int packedElementSize, int memory_size) { + const int totalMemSizeAligned = iAlignDown(memory_size, sizeof(TData)); + const int numElements = iDivDown(memory_size, sizeof(TData)); + + // Clean output buffer before current test + checkCudaErrors(cudaMemset(d_odata, 0, memory_size)); + // Run test + checkCudaErrors(cudaDeviceSynchronize()); + sdkResetTimer(&hTimer); + sdkStartTimer(&hTimer); + + for (int i = 0; i < NUM_ITERATIONS; i++) { + testKernel + <<<64, 256>>>((TData *)d_odata, (TData *)d_idata, numElements); + getLastCudaError("testKernel() execution failed\n"); + } + + checkCudaErrors(cudaDeviceSynchronize()); + sdkStopTimer(&hTimer); + double gpuTime = sdkGetTimerValue(&hTimer) / NUM_ITERATIONS; + printf("Avg. time: %f ms / Copy throughput: %f GB/s.\n", gpuTime, + (double)totalMemSizeAligned / (gpuTime * 0.001 * 1073741824.0)); + + // Read back GPU results and run validation + checkCudaErrors( + cudaMemcpy(h_odataGPU, d_odata, memory_size, cudaMemcpyDeviceToHost)); + int flag = testCPU((TData *)h_odataGPU, (TData *)h_idataCPU, numElements, + packedElementSize); + + printf(flag ? "\tTEST OK\n" : "\tTEST FAILURE\n"); + + return !flag; +} + +int main(int argc, char **argv) { + int i, nTotalFailures = 0; + + char* logDir = NULL; + if (checkCmdLineFlag(argc, (const char **)argv, "logDir")) { + getCmdLineArgumentString(argc, (const char **)argv, "logDir=", &logDir); + } + + sdkCreateTimer(&totalTimer); + sdkStartTimer(&totalTimer); + int devID; + cudaDeviceProp deviceProp; + printf("[%s] - Starting...\n", argv[0]); + + // find first CUDA device + devID = findCudaDevice(argc, (const char **)argv); + + // get number of SMs on this GPU + checkCudaErrors(cudaGetDeviceProperties(&deviceProp, devID)); + printf("[%s] has %d MP(s) x %d (Cores/MP) = %d (Cores)\n", deviceProp.name, + deviceProp.multiProcessorCount, + _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor), + _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) * + deviceProp.multiProcessorCount); + + // Anything that is less than 192 Cores will have a scaled down workload + float scale_factor = + max((192.0f / (_ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) * + (float)deviceProp.multiProcessorCount)), + 1.0f); + + int MemorySize = (int)(MEM_SIZE / scale_factor) & + 0xffffff00; // force multiple of 256 bytes + + printf("> Compute scaling value = %4.2f\n", scale_factor); + printf("> Memory Size = %d\n", MemorySize); + + // sdkCreateTimer(&totalTimer); + // sdkStartTimer(&totalTimer); + sdkCreateTimer(&hTimer); + + printf("Allocating memory...\n"); + h_idataCPU = (unsigned char *)malloc(MemorySize); + h_odataGPU = (unsigned char *)malloc(MemorySize); + checkCudaErrors(cudaMalloc((void **)&d_idata, MemorySize)); + checkCudaErrors(cudaMalloc((void **)&d_odata, MemorySize)); + + printf("Generating host input data array...\n"); + + for (i = 0; i < MemorySize; i++) { + h_idataCPU[i] = (i & 0xFF) + 1; + } + + printf("Uploading input data to GPU memory...\n"); + checkCudaErrors( + cudaMemcpy(d_idata, h_idataCPU, MemorySize, cudaMemcpyHostToDevice)); + + printf("Testing misaligned types...\n"); + printf("uint8...\n"); + nTotalFailures += runTest(1, MemorySize); + + printf("uint16...\n"); + nTotalFailures += runTest(2, MemorySize); + + printf("RGBA8_misaligned...\n"); + nTotalFailures += runTest(4, MemorySize); + + printf("LA32_misaligned...\n"); + nTotalFailures += runTest(8, MemorySize); + + printf("RGB32_misaligned...\n"); + nTotalFailures += runTest(12, MemorySize); + + printf("RGBA32_misaligned...\n"); + nTotalFailures += runTest(16, MemorySize); + + printf("Testing aligned types...\n"); + printf("RGBA8...\n"); + nTotalFailures += runTest(4, MemorySize); + + printf("I32...\n"); + nTotalFailures += runTest(4, MemorySize); + + printf("LA32...\n"); + nTotalFailures += runTest(8, MemorySize); + + printf("RGB32...\n"); + nTotalFailures += runTest(12, MemorySize); + + printf("RGBA32...\n"); + nTotalFailures += runTest(16, MemorySize); + + printf("RGBA32_2...\n"); + nTotalFailures += runTest(32, MemorySize); + + printf("\n[alignedTypes] -> Test Results: %d Failures\n", nTotalFailures); + + printf("Shutting down...\n"); + checkCudaErrors(cudaFree(d_idata)); + checkCudaErrors(cudaFree(d_odata)); + free(h_odataGPU); + free(h_idataCPU); + + sdkStopTimer(&totalTimer); + double totalTime = sdkGetTimerValue(&totalTimer); + printf("Total time: %f ms \n", totalTime); + if (logDir != NULL) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << totalTime << std::endl; + outFile.close(); + } + + sdkDeleteTimer(&hTimer); + sdkDeleteTimer(&totalTimer); + + + if (nTotalFailures != 0) { + printf("Test failed!\n"); + exit(EXIT_FAILURE); + } + + printf("Test passed\n"); + exit(EXIT_SUCCESS); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/asyncAPI.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/asyncAPI.cu new file mode 100644 index 0000000..1af3920 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/asyncAPI.cu @@ -0,0 +1,142 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This sample illustrates the usage of CUDA events for both GPU timing and + * overlapping CPU and GPU execution. Events are inserted into a stream + * of CUDA calls. Since CUDA stream calls are asynchronous, the CPU can + * perform computations while GPU is executing (including DMA memcopies + * between the host and device). CPU can query CUDA events to determine + * whether GPU has completed tasks. + */ + +// includes, system +#include + +// includes CUDA Runtime +#include +// #include + +// includes, project +#include +#include // helper utility functions + +__global__ void increment_kernel(int *g_data, int inc_value) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + g_data[idx] = g_data[idx] + inc_value; +} + +bool correct_output(int *data, const int n, const int x) { + for (int i = 0; i < n; i++) + if (data[i] != x) { + printf("Error! data[%d] = %d, ref = %d\n", i, data[i], x); + return false; + } + + return true; +} + +int main(int argc, char *argv[]) { + int devID; + cudaDeviceProp deviceProps; + + printf("[%s] - Starting...\n", argv[0]); + + // This will pick the best possible CUDA capable device + devID = findCudaDevice(argc, (const char **)argv); + + // get device name + checkCudaErrors(cudaGetDeviceProperties(&deviceProps, devID)); + printf("CUDA device [%s]\n", deviceProps.name); + + int n = 16 * 1024 * 1024; + int nbytes = n * sizeof(int); + int value = 26; + + // allocate host memory + int *a = 0; + checkCudaErrors(cudaMallocHost((void **)&a, nbytes)); + memset(a, 0, nbytes); + + // allocate device memory + int *d_a = 0; + checkCudaErrors(cudaMalloc((void **)&d_a, nbytes)); + checkCudaErrors(cudaMemset(d_a, 255, nbytes)); + + // set kernel launch configuration + dim3 threads = dim3(512, 1); + dim3 blocks = dim3(n / threads.x, 1); + + // create cuda event handles + cudaEvent_t start, stop; + checkCudaErrors(cudaEventCreate(&start)); + checkCudaErrors(cudaEventCreate(&stop)); + + StopWatchInterface *timer = NULL; + sdkCreateTimer(&timer); + sdkResetTimer(&timer); + + checkCudaErrors(cudaDeviceSynchronize()); + float gpu_time = 0.0f; + + // asynchronously issue work to the GPU (all to stream 0) + //checkCudaErrors(cudaProfilerStart()); + sdkStartTimer(&timer); + cudaEventRecord(start, 0); + cudaMemcpyAsync(d_a, a, nbytes, cudaMemcpyHostToDevice, 0); + increment_kernel<<>>(d_a, value); + cudaMemcpyAsync(a, d_a, nbytes, cudaMemcpyDeviceToHost, 0); + cudaEventRecord(stop, 0); + sdkStopTimer(&timer); + //checkCudaErrors(cudaProfilerStop()); + + // have CPU do some work while waiting for stage 1 to finish + unsigned long int counter = 0; + + while (cudaEventQuery(stop) == cudaErrorNotReady) { + counter++; + } + + checkCudaErrors(cudaEventElapsedTime(&gpu_time, start, stop)); + + // print the cpu and gpu times + printf("time spent executing by the GPU: %.2f\n", gpu_time); + printf("time spent by CPU in CUDA calls: %.2f\n", sdkGetTimerValue(&timer)); + printf("CPU executed %lu iterations while waiting for GPU to finish\n", + counter); + + // check the output for correctness + bool bFinalResults = correct_output(a, n, value); + + // release resources + checkCudaErrors(cudaEventDestroy(start)); + checkCudaErrors(cudaEventDestroy(stop)); + checkCudaErrors(cudaFreeHost(a)); + checkCudaErrors(cudaFree(d_a)); + + exit(bFinalResults ? EXIT_SUCCESS : EXIT_FAILURE); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/bandwidthTest.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/bandwidthTest.cu new file mode 100644 index 0000000..ae0fc2b --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/bandwidthTest.cu @@ -0,0 +1,969 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This is a simple test program to measure the memcopy bandwidth of the GPU. + * It can measure device to device copy bandwidth, host to device copy bandwidth + * for pageable and pinned memory, and device to host copy bandwidth for + * pageable and pinned memory. + * + * Usage: + * ./bandwidthTest [option]... + */ + +// CUDA runtime +#include + +// includes +#include // helper functions for CUDA error checking and initialization +#include // helper for shared functions common to CUDA Samples + +#include + +#include +#include +#include + +static const char *sSDKsample = "CUDA Bandwidth Test"; + +// defines, project +#define MEMCOPY_ITERATIONS 100 +#define DEFAULT_SIZE (32 * (1e6)) // 32 M +#define DEFAULT_INCREMENT (4 * (1e6)) // 4 M +#define CACHE_CLEAR_SIZE (16 * (1e6)) // 16 M + +// shmoo mode defines +#define SHMOO_MEMSIZE_MAX (64 * (1e6)) // 64 M +#define SHMOO_MEMSIZE_START (1e3) // 1 KB +#define SHMOO_INCREMENT_1KB (1e3) // 1 KB +#define SHMOO_INCREMENT_2KB (2 * 1e3) // 2 KB +#define SHMOO_INCREMENT_10KB (10 * (1e3)) // 10KB +#define SHMOO_INCREMENT_100KB (100 * (1e3)) // 100 KB +#define SHMOO_INCREMENT_1MB (1e6) // 1 MB +#define SHMOO_INCREMENT_2MB (2 * 1e6) // 2 MB +#define SHMOO_INCREMENT_4MB (4 * 1e6) // 4 MB +#define SHMOO_LIMIT_20KB (20 * (1e3)) // 20 KB +#define SHMOO_LIMIT_50KB (50 * (1e3)) // 50 KB +#define SHMOO_LIMIT_100KB (100 * (1e3)) // 100 KB +#define SHMOO_LIMIT_1MB (1e6) // 1 MB +#define SHMOO_LIMIT_16MB (16 * 1e6) // 16 MB +#define SHMOO_LIMIT_32MB (32 * 1e6) // 32 MB + +// CPU cache flush +#define FLUSH_SIZE (256 * 1024 * 1024) +char *flush_buf; + +// enums, project +enum testMode { QUICK_MODE, RANGE_MODE, SHMOO_MODE }; +enum memcpyKind { DEVICE_TO_HOST, HOST_TO_DEVICE, DEVICE_TO_DEVICE }; +enum printMode { USER_READABLE, CSV }; +enum memoryMode { PINNED, PAGEABLE }; + +const char *sMemoryCopyKind[] = {"Device to Host", "Host to Device", + "Device to Device", NULL}; + +const char *sMemoryMode[] = {"PINNED", "PAGEABLE", NULL}; + +// if true, use CPU based timing for everything +static bool bDontUseGPUTiming; + +int *pArgc = NULL; +char **pArgv = NULL; + +//////////////////////////////////////////////////////////////////////////////// +// declaration, forward +int runTest(const int argc, const char **argv); +void testBandwidth(unsigned int start, unsigned int end, unsigned int increment, + testMode mode, memcpyKind kind, printMode printmode, + memoryMode memMode, int startDevice, int endDevice, bool wc); +void testBandwidthQuick(unsigned int size, memcpyKind kind, printMode printmode, + memoryMode memMode, int startDevice, int endDevice, + bool wc); +void testBandwidthRange(unsigned int start, unsigned int end, + unsigned int increment, memcpyKind kind, + printMode printmode, memoryMode memMode, + int startDevice, int endDevice, bool wc); +void testBandwidthShmoo(memcpyKind kind, printMode printmode, + memoryMode memMode, int startDevice, int endDevice, + bool wc); +float testDeviceToHostTransfer(unsigned int memSize, memoryMode memMode, + bool wc); +float testHostToDeviceTransfer(unsigned int memSize, memoryMode memMode, + bool wc); +float testDeviceToDeviceTransfer(unsigned int memSize); +void printResultsReadable(unsigned int *memSizes, double *bandwidths, + unsigned int count, memcpyKind kind, + memoryMode memMode, int iNumDevs, bool wc); +void printResultsCSV(unsigned int *memSizes, double *bandwidths, + unsigned int count, memcpyKind kind, memoryMode memMode, + int iNumDevs, bool wc); +void printHelp(void); + +//////////////////////////////////////////////////////////////////////////////// +// Program main +//////////////////////////////////////////////////////////////////////////////// +int main(int argc, char **argv) { + pArgc = &argc; + pArgv = argv; + + flush_buf = (char *)malloc(FLUSH_SIZE); + + // set logfile name and start logs + printf("[%s] - Starting...\n", sSDKsample); + + int iRetVal = runTest(argc, (const char **)argv); + + if (iRetVal < 0) { + checkCudaErrors(cudaSetDevice(0)); + } + + // finish + printf("%s\n", (iRetVal == 0) ? "Result = PASS" : "Result = FAIL"); + + printf( + "\nNOTE: The CUDA Samples are not meant for performance measurements. " + "Results may vary when GPU Boost is enabled.\n"); + + free(flush_buf); + + exit((iRetVal == 0) ? EXIT_SUCCESS : EXIT_FAILURE); +} + +/////////////////////////////////////////////////////////////////////////////// +// Parse args, run the appropriate tests +/////////////////////////////////////////////////////////////////////////////// +int runTest(const int argc, const char **argv) { + int start = DEFAULT_SIZE; + int end = DEFAULT_SIZE; + int startDevice = 0; + int endDevice = 0; + int increment = DEFAULT_INCREMENT; + testMode mode = QUICK_MODE; + bool htod = false; + bool dtoh = false; + bool dtod = false; + bool wc = false; + char *modeStr; + char *device = NULL; + printMode printmode = USER_READABLE; + char *memModeStr = NULL; + memoryMode memMode = PINNED; + + // process command line args + if (checkCmdLineFlag(argc, argv, "help")) { + printHelp(); + return 0; + } + + if (checkCmdLineFlag(argc, argv, "csv")) { + printmode = CSV; + } + + if (getCmdLineArgumentString(argc, argv, "memory=", &memModeStr)) { + if (strcmp(memModeStr, "pageable") == 0) { + memMode = PAGEABLE; + } else if (strcmp(memModeStr, "pinned") == 0) { + memMode = PINNED; + } else { + printf("Invalid memory mode - valid modes are pageable or pinned\n"); + printf("See --help for more information\n"); + return -1000; + } + } else { + // default - pinned memory + memMode = PINNED; + } + + if (getCmdLineArgumentString(argc, argv, "device=", &device)) { + int deviceCount; + cudaError_t error_id = cudaGetDeviceCount(&deviceCount); + + if (error_id != cudaSuccess) { + printf("cudaGetDeviceCount returned %d\n-> %s\n", (int)error_id, + cudaGetErrorString(error_id)); + exit(EXIT_FAILURE); + } + + if (deviceCount == 0) { + printf("!!!!!No devices found!!!!!\n"); + return -2000; + } + + if (strcmp(device, "all") == 0) { + printf( + "\n!!!!!Cumulative Bandwidth to be computed from all the devices " + "!!!!!!\n\n"); + startDevice = 0; + endDevice = deviceCount - 1; + } else { + startDevice = endDevice = atoi(device); + + if (startDevice >= deviceCount || startDevice < 0) { + printf( + "\n!!!!!Invalid GPU number %d given hence default gpu %d will be " + "used !!!!!\n", + startDevice, 0); + startDevice = endDevice = 0; + } + } + } + + printf("Running on...\n\n"); + + for (int currentDevice = startDevice; currentDevice <= endDevice; + currentDevice++) { + cudaDeviceProp deviceProp; + cudaError_t error_id = cudaGetDeviceProperties(&deviceProp, currentDevice); + + if (error_id == cudaSuccess) { + printf(" Device %d: %s\n", currentDevice, deviceProp.name); + + if (deviceProp.computeMode == cudaComputeModeProhibited) { + fprintf(stderr, + "Error: device is running in , no " + "threads can use ::cudaSetDevice().\n"); + checkCudaErrors(cudaSetDevice(currentDevice)); + + exit(EXIT_FAILURE); + } + } else { + printf("cudaGetDeviceProperties returned %d\n-> %s\n", (int)error_id, + cudaGetErrorString(error_id)); + checkCudaErrors(cudaSetDevice(currentDevice)); + + exit(EXIT_FAILURE); + } + } + + if (getCmdLineArgumentString(argc, argv, "mode=", &modeStr)) { + // figure out the mode + if (strcmp(modeStr, "quick") == 0) { + printf(" Quick Mode\n\n"); + mode = QUICK_MODE; + } else if (strcmp(modeStr, "shmoo") == 0) { + printf(" Shmoo Mode\n\n"); + mode = SHMOO_MODE; + } else if (strcmp(modeStr, "range") == 0) { + printf(" Range Mode\n\n"); + mode = RANGE_MODE; + } else { + printf("Invalid mode - valid modes are quick, range, or shmoo\n"); + printf("See --help for more information\n"); + return -3000; + } + } else { + // default mode - quick + printf(" Quick Mode\n\n"); + mode = QUICK_MODE; + } + + if (checkCmdLineFlag(argc, argv, "htod")) { + htod = true; + } + + if (checkCmdLineFlag(argc, argv, "dtoh")) { + dtoh = true; + } + + if (checkCmdLineFlag(argc, argv, "dtod")) { + dtod = true; + } + +#if CUDART_VERSION >= 2020 + + if (checkCmdLineFlag(argc, argv, "wc")) { + wc = true; + } + +#endif + + if (checkCmdLineFlag(argc, argv, "cputiming")) { + bDontUseGPUTiming = true; + } + + if (!htod && !dtoh && !dtod) { + // default: All + htod = true; + dtoh = true; + dtod = true; + } + + if (RANGE_MODE == mode) { + if (checkCmdLineFlag(argc, (const char **)argv, "start")) { + start = getCmdLineArgumentInt(argc, argv, "start"); + + if (start <= 0) { + printf("Illegal argument - start must be greater than zero\n"); + return -4000; + } + } else { + printf("Must specify a starting size in range mode\n"); + printf("See --help for more information\n"); + return -5000; + } + + if (checkCmdLineFlag(argc, (const char **)argv, "end")) { + end = getCmdLineArgumentInt(argc, argv, "end"); + + if (end <= 0) { + printf("Illegal argument - end must be greater than zero\n"); + return -6000; + } + + if (start > end) { + printf("Illegal argument - start is greater than end\n"); + return -7000; + } + } else { + printf("Must specify an end size in range mode.\n"); + printf("See --help for more information\n"); + return -8000; + } + + if (checkCmdLineFlag(argc, argv, "increment")) { + increment = getCmdLineArgumentInt(argc, argv, "increment"); + + if (increment <= 0) { + printf("Illegal argument - increment must be greater than zero\n"); + return -9000; + } + } else { + printf("Must specify an increment in user mode\n"); + printf("See --help for more information\n"); + return -10000; + } + } + + if (htod) { + testBandwidth((unsigned int)start, (unsigned int)end, + (unsigned int)increment, mode, HOST_TO_DEVICE, printmode, + memMode, startDevice, endDevice, wc); + } + + if (dtoh) { + testBandwidth((unsigned int)start, (unsigned int)end, + (unsigned int)increment, mode, DEVICE_TO_HOST, printmode, + memMode, startDevice, endDevice, wc); + } + + if (dtod) { + testBandwidth((unsigned int)start, (unsigned int)end, + (unsigned int)increment, mode, DEVICE_TO_DEVICE, printmode, + memMode, startDevice, endDevice, wc); + } + + // Ensure that we reset all CUDA Devices in question + for (int nDevice = startDevice; nDevice <= endDevice; nDevice++) { + cudaSetDevice(nDevice); + } + + return 0; +} + +/////////////////////////////////////////////////////////////////////////////// +// Run a bandwidth test +/////////////////////////////////////////////////////////////////////////////// +void testBandwidth(unsigned int start, unsigned int end, unsigned int increment, + testMode mode, memcpyKind kind, printMode printmode, + memoryMode memMode, int startDevice, int endDevice, + bool wc) { + switch (mode) { + case QUICK_MODE: + testBandwidthQuick(DEFAULT_SIZE, kind, printmode, memMode, startDevice, + endDevice, wc); + break; + + case RANGE_MODE: + testBandwidthRange(start, end, increment, kind, printmode, memMode, + startDevice, endDevice, wc); + break; + + case SHMOO_MODE: + testBandwidthShmoo(kind, printmode, memMode, startDevice, endDevice, wc); + break; + + default: + break; + } +} + +////////////////////////////////////////////////////////////////////// +// Run a quick mode bandwidth test +////////////////////////////////////////////////////////////////////// +void testBandwidthQuick(unsigned int size, memcpyKind kind, printMode printmode, + memoryMode memMode, int startDevice, int endDevice, + bool wc) { + testBandwidthRange(size, size, DEFAULT_INCREMENT, kind, printmode, memMode, + startDevice, endDevice, wc); +} + +/////////////////////////////////////////////////////////////////////// +// Run a range mode bandwidth test +////////////////////////////////////////////////////////////////////// +void testBandwidthRange(unsigned int start, unsigned int end, + unsigned int increment, memcpyKind kind, + printMode printmode, memoryMode memMode, + int startDevice, int endDevice, bool wc) { + // count the number of copies we're going to run + unsigned int count = 1 + ((end - start) / increment); + + unsigned int *memSizes = (unsigned int *)malloc(count * sizeof(unsigned int)); + double *bandwidths = (double *)malloc(count * sizeof(double)); + + // Before calculating the cumulative bandwidth, initialize bandwidths array to + // NULL + for (unsigned int i = 0; i < count; i++) { + bandwidths[i] = 0.0; + } + + // Use the device asked by the user + for (int currentDevice = startDevice; currentDevice <= endDevice; + currentDevice++) { + cudaSetDevice(currentDevice); + + // run each of the copies + for (unsigned int i = 0; i < count; i++) { + memSizes[i] = start + i * increment; + + switch (kind) { + case DEVICE_TO_HOST: + bandwidths[i] += testDeviceToHostTransfer(memSizes[i], memMode, wc); + break; + + case HOST_TO_DEVICE: + bandwidths[i] += testHostToDeviceTransfer(memSizes[i], memMode, wc); + break; + + case DEVICE_TO_DEVICE: + bandwidths[i] += testDeviceToDeviceTransfer(memSizes[i]); + break; + } + } + } // Complete the bandwidth computation on all the devices + + // print results + if (printmode == CSV) { + printResultsCSV(memSizes, bandwidths, count, kind, memMode, + (1 + endDevice - startDevice), wc); + } else { + printResultsReadable(memSizes, bandwidths, count, kind, memMode, + (1 + endDevice - startDevice), wc); + } + + // clean up + free(memSizes); + free(bandwidths); +} + +////////////////////////////////////////////////////////////////////////////// +// Intense shmoo mode - covers a large range of values with varying increments +////////////////////////////////////////////////////////////////////////////// +void testBandwidthShmoo(memcpyKind kind, printMode printmode, + memoryMode memMode, int startDevice, int endDevice, + bool wc) { + // count the number of copies to make + unsigned int count = + 1 + (SHMOO_LIMIT_20KB / SHMOO_INCREMENT_1KB) + + ((SHMOO_LIMIT_50KB - SHMOO_LIMIT_20KB) / SHMOO_INCREMENT_2KB) + + ((SHMOO_LIMIT_100KB - SHMOO_LIMIT_50KB) / SHMOO_INCREMENT_10KB) + + ((SHMOO_LIMIT_1MB - SHMOO_LIMIT_100KB) / SHMOO_INCREMENT_100KB) + + ((SHMOO_LIMIT_16MB - SHMOO_LIMIT_1MB) / SHMOO_INCREMENT_1MB) + + ((SHMOO_LIMIT_32MB - SHMOO_LIMIT_16MB) / SHMOO_INCREMENT_2MB) + + ((SHMOO_MEMSIZE_MAX - SHMOO_LIMIT_32MB) / SHMOO_INCREMENT_4MB); + + unsigned int *memSizes = (unsigned int *)malloc(count * sizeof(unsigned int)); + double *bandwidths = (double *)malloc(count * sizeof(double)); + + // Before calculating the cumulative bandwidth, initialize bandwidths array to + // NULL + for (unsigned int i = 0; i < count; i++) { + bandwidths[i] = 0.0; + } + + // Use the device asked by the user + for (int currentDevice = startDevice; currentDevice <= endDevice; + currentDevice++) { + cudaSetDevice(currentDevice); + // Run the shmoo + int iteration = 0; + unsigned int memSize = 0; + + while (memSize <= SHMOO_MEMSIZE_MAX) { + if (memSize < SHMOO_LIMIT_20KB) { + memSize += SHMOO_INCREMENT_1KB; + } else if (memSize < SHMOO_LIMIT_50KB) { + memSize += SHMOO_INCREMENT_2KB; + } else if (memSize < SHMOO_LIMIT_100KB) { + memSize += SHMOO_INCREMENT_10KB; + } else if (memSize < SHMOO_LIMIT_1MB) { + memSize += SHMOO_INCREMENT_100KB; + } else if (memSize < SHMOO_LIMIT_16MB) { + memSize += SHMOO_INCREMENT_1MB; + } else if (memSize < SHMOO_LIMIT_32MB) { + memSize += SHMOO_INCREMENT_2MB; + } else { + memSize += SHMOO_INCREMENT_4MB; + } + + memSizes[iteration] = memSize; + + switch (kind) { + case DEVICE_TO_HOST: + bandwidths[iteration] += + testDeviceToHostTransfer(memSizes[iteration], memMode, wc); + break; + + case HOST_TO_DEVICE: + bandwidths[iteration] += + testHostToDeviceTransfer(memSizes[iteration], memMode, wc); + break; + + case DEVICE_TO_DEVICE: + bandwidths[iteration] += + testDeviceToDeviceTransfer(memSizes[iteration]); + break; + } + + iteration++; + printf("."); + fflush(0); + } + } // Complete the bandwidth computation on all the devices + + // print results + printf("\n"); + + if (CSV == printmode) { + printResultsCSV(memSizes, bandwidths, count, kind, memMode, + (1 + endDevice - startDevice), wc); + } else { + printResultsReadable(memSizes, bandwidths, count, kind, memMode, + (1 + endDevice - startDevice), wc); + } + + // clean up + free(memSizes); + free(bandwidths); +} + +/////////////////////////////////////////////////////////////////////////////// +// test the bandwidth of a device to host memcopy of a specific size +/////////////////////////////////////////////////////////////////////////////// +float testDeviceToHostTransfer(unsigned int memSize, memoryMode memMode, + bool wc) { + StopWatchInterface *timer = NULL; + float elapsedTimeInMs = 0.0f; + float bandwidthInGBs = 0.0f; + unsigned char *h_idata = NULL; + unsigned char *h_odata = NULL; + cudaEvent_t start, stop; + + sdkCreateTimer(&timer); + checkCudaErrors(cudaEventCreate(&start)); + checkCudaErrors(cudaEventCreate(&stop)); + + // allocate host memory + if (PINNED == memMode) { + // pinned memory mode - use special function to get OS-pinned memory +#if CUDART_VERSION >= 2020 + checkCudaErrors(cudaHostAlloc((void **)&h_idata, memSize, + (wc) ? cudaHostAllocWriteCombined : 0)); + checkCudaErrors(cudaHostAlloc((void **)&h_odata, memSize, + (wc) ? cudaHostAllocWriteCombined : 0)); +#else + checkCudaErrors(cudaMallocHost((void **)&h_idata, memSize)); + checkCudaErrors(cudaMallocHost((void **)&h_odata, memSize)); +#endif + } else { + // pageable memory mode - use malloc + h_idata = (unsigned char *)malloc(memSize); + h_odata = (unsigned char *)malloc(memSize); + + if (h_idata == 0 || h_odata == 0) { + fprintf(stderr, "Not enough memory avaialable on host to run test!\n"); + exit(EXIT_FAILURE); + } + } + + // initialize the memory + for (unsigned int i = 0; i < memSize / sizeof(unsigned char); i++) { + h_idata[i] = (unsigned char)(i & 0xff); + } + + // allocate device memory + unsigned char *d_idata; + checkCudaErrors(cudaMalloc((void **)&d_idata, memSize)); + + // initialize the device memory + checkCudaErrors( + cudaMemcpy(d_idata, h_idata, memSize, cudaMemcpyHostToDevice)); + + // copy data from GPU to Host + if (PINNED == memMode) { + if (bDontUseGPUTiming) sdkStartTimer(&timer); + checkCudaErrors(cudaEventRecord(start, 0)); + for (unsigned int i = 0; i < MEMCOPY_ITERATIONS; i++) { + checkCudaErrors(cudaMemcpyAsync(h_odata, d_idata, memSize, + cudaMemcpyDeviceToHost, 0)); + } + checkCudaErrors(cudaEventRecord(stop, 0)); + checkCudaErrors(cudaDeviceSynchronize()); + checkCudaErrors(cudaEventElapsedTime(&elapsedTimeInMs, start, stop)); + if (bDontUseGPUTiming) { + sdkStopTimer(&timer); + elapsedTimeInMs = sdkGetTimerValue(&timer); + sdkResetTimer(&timer); + } + } else { + elapsedTimeInMs = 0; + for (unsigned int i = 0; i < MEMCOPY_ITERATIONS; i++) { + sdkStartTimer(&timer); + checkCudaErrors( + cudaMemcpy(h_odata, d_idata, memSize, cudaMemcpyDeviceToHost)); + sdkStopTimer(&timer); + elapsedTimeInMs += sdkGetTimerValue(&timer); + sdkResetTimer(&timer); + memset(flush_buf, i, FLUSH_SIZE); + } + } + + // calculate bandwidth in GB/s + double time_s = elapsedTimeInMs / 1e3; + bandwidthInGBs = (memSize * (float)MEMCOPY_ITERATIONS) / (double)1e9; + bandwidthInGBs = bandwidthInGBs / time_s; + // clean up memory + checkCudaErrors(cudaEventDestroy(stop)); + checkCudaErrors(cudaEventDestroy(start)); + sdkDeleteTimer(&timer); + + if (PINNED == memMode) { + checkCudaErrors(cudaFreeHost(h_idata)); + checkCudaErrors(cudaFreeHost(h_odata)); + } else { + free(h_idata); + free(h_odata); + } + + checkCudaErrors(cudaFree(d_idata)); + + return bandwidthInGBs; +} + +/////////////////////////////////////////////////////////////////////////////// +//! test the bandwidth of a host to device memcopy of a specific size +/////////////////////////////////////////////////////////////////////////////// +float testHostToDeviceTransfer(unsigned int memSize, memoryMode memMode, + bool wc) { + StopWatchInterface *timer = NULL; + float elapsedTimeInMs = 0.0f; + float bandwidthInGBs = 0.0f; + cudaEvent_t start, stop; + sdkCreateTimer(&timer); + checkCudaErrors(cudaEventCreate(&start)); + checkCudaErrors(cudaEventCreate(&stop)); + + // allocate host memory + unsigned char *h_odata = NULL; + + if (PINNED == memMode) { +#if CUDART_VERSION >= 2020 + // pinned memory mode - use special function to get OS-pinned memory + checkCudaErrors(cudaHostAlloc((void **)&h_odata, memSize, + (wc) ? cudaHostAllocWriteCombined : 0)); +#else + // pinned memory mode - use special function to get OS-pinned memory + checkCudaErrors(cudaMallocHost((void **)&h_odata, memSize)); +#endif + } else { + // pageable memory mode - use malloc + h_odata = (unsigned char *)malloc(memSize); + + if (h_odata == 0) { + fprintf(stderr, "Not enough memory available on host to run test!\n"); + exit(EXIT_FAILURE); + } + } + + unsigned char *h_cacheClear1 = (unsigned char *)malloc(CACHE_CLEAR_SIZE); + unsigned char *h_cacheClear2 = (unsigned char *)malloc(CACHE_CLEAR_SIZE); + + if (h_cacheClear1 == 0 || h_cacheClear2 == 0) { + fprintf(stderr, "Not enough memory available on host to run test!\n"); + exit(EXIT_FAILURE); + } + + // initialize the memory + for (unsigned int i = 0; i < memSize / sizeof(unsigned char); i++) { + h_odata[i] = (unsigned char)(i & 0xff); + } + + for (unsigned int i = 0; i < CACHE_CLEAR_SIZE / sizeof(unsigned char); i++) { + h_cacheClear1[i] = (unsigned char)(i & 0xff); + h_cacheClear2[i] = (unsigned char)(0xff - (i & 0xff)); + } + + // allocate device memory + unsigned char *d_idata; + checkCudaErrors(cudaMalloc((void **)&d_idata, memSize)); + + // copy host memory to device memory + if (PINNED == memMode) { + if (bDontUseGPUTiming) sdkStartTimer(&timer); + checkCudaErrors(cudaEventRecord(start, 0)); + for (unsigned int i = 0; i < MEMCOPY_ITERATIONS; i++) { + checkCudaErrors(cudaMemcpyAsync(d_idata, h_odata, memSize, + cudaMemcpyHostToDevice, 0)); + } + checkCudaErrors(cudaEventRecord(stop, 0)); + checkCudaErrors(cudaDeviceSynchronize()); + checkCudaErrors(cudaEventElapsedTime(&elapsedTimeInMs, start, stop)); + if (bDontUseGPUTiming) { + sdkStopTimer(&timer); + elapsedTimeInMs = sdkGetTimerValue(&timer); + sdkResetTimer(&timer); + } + } else { + elapsedTimeInMs = 0; + for (unsigned int i = 0; i < MEMCOPY_ITERATIONS; i++) { + sdkStartTimer(&timer); + checkCudaErrors( + cudaMemcpy(d_idata, h_odata, memSize, cudaMemcpyHostToDevice)); + sdkStopTimer(&timer); + elapsedTimeInMs += sdkGetTimerValue(&timer); + sdkResetTimer(&timer); + memset(flush_buf, i, FLUSH_SIZE); + } + } + + // calculate bandwidth in GB/s + double time_s = elapsedTimeInMs / 1e3; + bandwidthInGBs = (memSize * (float)MEMCOPY_ITERATIONS) / (double)1e9; + bandwidthInGBs = bandwidthInGBs / time_s; + // clean up memory + checkCudaErrors(cudaEventDestroy(stop)); + checkCudaErrors(cudaEventDestroy(start)); + sdkDeleteTimer(&timer); + + if (PINNED == memMode) { + checkCudaErrors(cudaFreeHost(h_odata)); + } else { + free(h_odata); + } + + free(h_cacheClear1); + free(h_cacheClear2); + checkCudaErrors(cudaFree(d_idata)); + + return bandwidthInGBs; +} + +/////////////////////////////////////////////////////////////////////////////// +//! test the bandwidth of a device to device memcopy of a specific size +/////////////////////////////////////////////////////////////////////////////// +float testDeviceToDeviceTransfer(unsigned int memSize) { + StopWatchInterface *timer = NULL; + float elapsedTimeInMs = 0.0f; + float bandwidthInGBs = 0.0f; + cudaEvent_t start, stop; + + sdkCreateTimer(&timer); + checkCudaErrors(cudaEventCreate(&start)); + checkCudaErrors(cudaEventCreate(&stop)); + + // allocate host memory + unsigned char *h_idata = (unsigned char *)malloc(memSize); + + if (h_idata == 0) { + fprintf(stderr, "Not enough memory avaialable on host to run test!\n"); + exit(EXIT_FAILURE); + } + + // initialize the host memory + for (unsigned int i = 0; i < memSize / sizeof(unsigned char); i++) { + h_idata[i] = (unsigned char)(i & 0xff); + } + + // allocate device memory + unsigned char *d_idata; + checkCudaErrors(cudaMalloc((void **)&d_idata, memSize)); + unsigned char *d_odata; + checkCudaErrors(cudaMalloc((void **)&d_odata, memSize)); + + // initialize memory + checkCudaErrors( + cudaMemcpy(d_idata, h_idata, memSize, cudaMemcpyHostToDevice)); + + // run the memcopy + sdkStartTimer(&timer); + checkCudaErrors(cudaEventRecord(start, 0)); + + for (unsigned int i = 0; i < MEMCOPY_ITERATIONS; i++) { + checkCudaErrors( + cudaMemcpy(d_odata, d_idata, memSize, cudaMemcpyDeviceToDevice)); + } + + checkCudaErrors(cudaEventRecord(stop, 0)); + + // Since device to device memory copies are non-blocking, + // cudaDeviceSynchronize() is required in order to get + // proper timing. + checkCudaErrors(cudaDeviceSynchronize()); + + // get the total elapsed time in ms + sdkStopTimer(&timer); + checkCudaErrors(cudaEventElapsedTime(&elapsedTimeInMs, start, stop)); + + if (bDontUseGPUTiming) { + elapsedTimeInMs = sdkGetTimerValue(&timer); + } + + // calculate bandwidth in GB/s + double time_s = elapsedTimeInMs / 1e3; + bandwidthInGBs = (2.0f * memSize * (float)MEMCOPY_ITERATIONS) / (double)1e9; + bandwidthInGBs = bandwidthInGBs / time_s; + + // clean up memory + sdkDeleteTimer(&timer); + free(h_idata); + checkCudaErrors(cudaEventDestroy(stop)); + checkCudaErrors(cudaEventDestroy(start)); + checkCudaErrors(cudaFree(d_idata)); + checkCudaErrors(cudaFree(d_odata)); + + return bandwidthInGBs; +} + +///////////////////////////////////////////////////////// +// print results in an easily read format +//////////////////////////////////////////////////////// +void printResultsReadable(unsigned int *memSizes, double *bandwidths, + unsigned int count, memcpyKind kind, + memoryMode memMode, int iNumDevs, bool wc) { + printf(" %s Bandwidth, %i Device(s)\n", sMemoryCopyKind[kind], iNumDevs); + printf(" %s Memory Transfers\n", sMemoryMode[memMode]); + + if (wc) { + printf(" Write-Combined Memory Writes are Enabled"); + } + + printf(" Transfer Size (Bytes)\tBandwidth(GB/s)\n"); + unsigned int i; + + for (i = 0; i < (count - 1); i++) { + printf(" %u\t\t\t%s%.1f\n", memSizes[i], + (memSizes[i] < 10000) ? "\t" : "", bandwidths[i]); + } + + printf(" %u\t\t\t%s%.1f\n\n", memSizes[i], + (memSizes[i] < 10000) ? "\t" : "", bandwidths[i]); +} + +/////////////////////////////////////////////////////////////////////////// +// print results in a database format +/////////////////////////////////////////////////////////////////////////// +void printResultsCSV(unsigned int *memSizes, double *bandwidths, + unsigned int count, memcpyKind kind, memoryMode memMode, + int iNumDevs, bool wc) { + std::string sConfig; + + // log config information + if (kind == DEVICE_TO_DEVICE) { + sConfig += "D2D"; + } else { + if (kind == DEVICE_TO_HOST) { + sConfig += "D2H"; + } else if (kind == HOST_TO_DEVICE) { + sConfig += "H2D"; + } + + if (memMode == PAGEABLE) { + sConfig += "-Paged"; + } else if (memMode == PINNED) { + sConfig += "-Pinned"; + + if (wc) { + sConfig += "-WriteCombined"; + } + } + } + + unsigned int i; + double dSeconds = 0.0; + + for (i = 0; i < count; i++) { + dSeconds = (double)memSizes[i] / (bandwidths[i] * (double)(1e9)); + printf( + "bandwidthTest-%s, Bandwidth = %.1f GB/s, Time = %.5f s, Size = %u " + "bytes, NumDevsUsed = %d\n", + sConfig.c_str(), bandwidths[i], dSeconds, memSizes[i], iNumDevs); + } +} + +/////////////////////////////////////////////////////////////////////////// +// Print help screen +/////////////////////////////////////////////////////////////////////////// +void printHelp(void) { + printf("Usage: bandwidthTest [OPTION]...\n"); + printf( + "Test the bandwidth for device to host, host to device, and device to " + "device transfers\n"); + printf("\n"); + printf( + "Example: measure the bandwidth of device to host pinned memory copies " + "in the range 1024 Bytes to 102400 Bytes in 1024 Byte increments\n"); + printf( + "./bandwidthTest --memory=pinned --mode=range --start=1024 --end=102400 " + "--increment=1024 --dtoh\n"); + + printf("\n"); + printf("Options:\n"); + printf("--help\tDisplay this help menu\n"); + printf("--csv\tPrint results as a CSV\n"); + printf("--device=[deviceno]\tSpecify the device device to be used\n"); + printf(" all - compute cumulative bandwidth on all the devices\n"); + printf(" 0,1,2,...,n - Specify any particular device to be used\n"); + printf("--memory=[MEMMODE]\tSpecify which memory mode to use\n"); + printf(" pageable - pageable memory\n"); + printf(" pinned - non-pageable system memory\n"); + printf("--mode=[MODE]\tSpecify the mode to use\n"); + printf(" quick - performs a quick measurement\n"); + printf(" range - measures a user-specified range of values\n"); + printf(" shmoo - performs an intense shmoo of a large range of values\n"); + + printf("--htod\tMeasure host to device transfers\n"); + printf("--dtoh\tMeasure device to host transfers\n"); + printf("--dtod\tMeasure device to device transfers\n"); +#if CUDART_VERSION >= 2020 + printf("--wc\tAllocate pinned memory as write-combined\n"); +#endif + printf("--cputiming\tForce CPU-based timing always\n"); + + printf("Range mode options\n"); + printf("--start=[SIZE]\tStarting transfer size in bytes\n"); + printf("--end=[SIZE]\tEnding transfer size in bytes\n"); + printf("--increment=[SIZE]\tIncrement size in bytes\n"); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/clock.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/clock.cu new file mode 100644 index 0000000..bb97e1b --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/clock.cu @@ -0,0 +1,178 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This example shows how to use the clock function to measure the performance + * of block of threads of a kernel accurately. Blocks are executed in parallel + * and out of order. Since there's no synchronization mechanism between blocks, + * we measure the clock once for each block. The clock samples are written to + * device memory. + */ + +// System includes +#include +#include +#include +#include + +// CUDA runtime +#include + +// helper functions and utilities to work with CUDA +#include +#include + +// This kernel computes a standard parallel reduction and evaluates the +// time it takes to do that for each block. The timing results are stored +// in device memory. +__global__ static void timedReduction(const float *input, float *output, + clock_t *timer) { + // __shared__ float shared[2 * blockDim.x]; + extern __shared__ float shared[]; + + const int tid = threadIdx.x; + const int bid = blockIdx.x; + + if (tid == 0) timer[bid] = clock(); + + // Copy input. + shared[tid] = input[tid]; + shared[tid + blockDim.x] = input[tid + blockDim.x]; + + // Perform reduction to find minimum. + for (int d = blockDim.x; d > 0; d /= 2) { + __syncthreads(); + + if (tid < d) { + float f0 = shared[tid]; + float f1 = shared[tid + d]; + + if (f1 < f0) { + shared[tid] = f1; + } + } + } + + // Write result. + if (tid == 0) output[bid] = shared[0]; + + __syncthreads(); + + if (tid == 0) timer[bid + gridDim.x] = clock(); +} + +#define NUM_BLOCKS 64 +#define NUM_THREADS 256 + +// It's interesting to change the number of blocks and the number of threads to +// understand how to keep the hardware busy. +// +// Here are some numbers I get on my G80: +// blocks - clocks +// 1 - 3096 +// 8 - 3232 +// 16 - 3364 +// 32 - 4615 +// 64 - 9981 +// +// With less than 16 blocks some of the multiprocessors of the device are idle. +// With more than 16 you are using all the multiprocessors, but there's only one +// block per multiprocessor and that doesn't allow you to hide the latency of +// the memory. With more than 32 the speed scales linearly. + +// Start the main CUDA Sample here + +StopWatchInterface *totalTimer = NULL; + +int main(int argc, char **argv) { + + char* logDir = NULL; + if (checkCmdLineFlag(argc, (const char **)argv, "logDir")) { + getCmdLineArgumentString(argc, (const char **)argv, "logDir=", &logDir); + } + + sdkCreateTimer(&totalTimer); + sdkStartTimer(&totalTimer); + + printf("CUDA Clock sample\n"); + + // This will pick the best possible CUDA capable device + int dev = findCudaDevice(argc, (const char **)argv); + + float *dinput = NULL; + float *doutput = NULL; + clock_t *dtimer = NULL; + + clock_t timer[NUM_BLOCKS * 2]; + float input[NUM_THREADS * 2]; + + for (int i = 0; i < NUM_THREADS * 2; i++) { + input[i] = (float)i; + } + + checkCudaErrors( + cudaMalloc((void **)&dinput, sizeof(float) * NUM_THREADS * 2)); + checkCudaErrors(cudaMalloc((void **)&doutput, sizeof(float) * NUM_BLOCKS)); + checkCudaErrors( + cudaMalloc((void **)&dtimer, sizeof(clock_t) * NUM_BLOCKS * 2)); + + checkCudaErrors(cudaMemcpy(dinput, input, sizeof(float) * NUM_THREADS * 2, + cudaMemcpyHostToDevice)); + + timedReduction<<>>( + dinput, doutput, dtimer); + + checkCudaErrors(cudaMemcpy(timer, dtimer, sizeof(clock_t) * NUM_BLOCKS * 2, + cudaMemcpyDeviceToHost)); + + checkCudaErrors(cudaFree(dinput)); + checkCudaErrors(cudaFree(doutput)); + checkCudaErrors(cudaFree(dtimer)); + + long double avgElapsedClocks = 0; + + for (int i = 0; i < NUM_BLOCKS; i++) { + avgElapsedClocks += (long double)(timer[i + NUM_BLOCKS] - timer[i]); + } + + avgElapsedClocks = avgElapsedClocks / NUM_BLOCKS; + printf("Average clocks/block = %Lf\n", avgElapsedClocks); + + sdkStopTimer(&totalTimer); + double totalTime = sdkGetTimerValue(&totalTimer); + printf("Total time: %f ms \n", totalTime); + if (logDir != NULL) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << totalTime << std::endl; + outFile.close(); + } + + sdkDeleteTimer(&totalTimer); + + + return EXIT_SUCCESS; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/convolutionSeparable.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/convolutionSeparable.cu new file mode 100644 index 0000000..1accf1a --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/convolutionSeparable.cu @@ -0,0 +1,182 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* +* This sample implements a separable convolution filter +* of a 2D image with an arbitrary kernel. +*/ + + +// Utilities and system includes +#include +#include +#include + +#include "./include/convolutionSeparable_common.cuh" +StopWatchInterface *totalTimer = NULL; +//////////////////////////////////////////////////////////////////////////////// +// Main program +//////////////////////////////////////////////////////////////////////////////// +int main(int argc, char **argv) { + cudaDeviceSynchronize(); + char* logDir = NULL; + if (checkCmdLineFlag(argc, (const char **)argv, "logDir")) { + getCmdLineArgumentString(argc, (const char **)argv, "logDir=", &logDir); + } + + sdkCreateTimer(&totalTimer); + sdkStartTimer(&totalTimer); + // start logs + printf("[%s] - Starting...\n", argv[0]); + + float *h_Kernel, *h_Input, *h_Buffer, *h_OutputCPU, *h_OutputGPU; + + float *d_Input, *d_Output, *d_Buffer; + + const int imageW = 3072; + const int imageH = 3072; + const int iterations = 16; + + StopWatchInterface *hTimer = NULL; + + // Use command-line specified CUDA device, otherwise use device with highest + // Gflops/s + findCudaDevice(argc, (const char **)argv); + + sdkCreateTimer(&hTimer); + + printf("Image Width x Height = %i x %i\n\n", imageW, imageH); + printf("Allocating and initializing host arrays...\n"); + h_Kernel = (float *)malloc(KERNEL_LENGTH * sizeof(float)); + h_Input = (float *)malloc(imageW * imageH * sizeof(float)); + h_Buffer = (float *)malloc(imageW * imageH * sizeof(float)); + h_OutputCPU = (float *)malloc(imageW * imageH * sizeof(float)); + h_OutputGPU = (float *)malloc(imageW * imageH * sizeof(float)); + srand(200); + + for (unsigned int i = 0; i < KERNEL_LENGTH; i++) { + h_Kernel[i] = (float)(rand() % 16); + } + + for (unsigned i = 0; i < imageW * imageH; i++) { + h_Input[i] = (float)(rand() % 16); + } + + printf("Allocating and initializing CUDA arrays...\n"); + checkCudaErrors( + cudaMalloc((void **)&d_Input, imageW * imageH * sizeof(float))); + checkCudaErrors( + cudaMalloc((void **)&d_Output, imageW * imageH * sizeof(float))); + checkCudaErrors( + cudaMalloc((void **)&d_Buffer, imageW * imageH * sizeof(float))); + + setConvolutionKernel(h_Kernel); + checkCudaErrors(cudaMemcpy(d_Input, h_Input, imageW * imageH * sizeof(float), + cudaMemcpyHostToDevice)); + + printf("Running GPU convolution (%u identical iterations)...\n\n", + iterations); + + for (int i = -1; i < iterations; i++) { + // i == -1 -- warmup iteration + if (i == 0) { + checkCudaErrors(cudaDeviceSynchronize()); + sdkResetTimer(&hTimer); + sdkStartTimer(&hTimer); + } + + convolutionRowsGPU(d_Buffer, d_Input, imageW, imageH); + + convolutionColumnsGPU(d_Output, d_Buffer, imageW, imageH); + } + + checkCudaErrors(cudaDeviceSynchronize()); + sdkStopTimer(&hTimer); + double gpuTime = 0.001 * sdkGetTimerValue(&hTimer) / (double)iterations; + printf( + "convolutionSeparable, Throughput = %.4f MPixels/sec, Time = %.5f s, " + "Size = %u Pixels, NumDevsUsed = %i, Workgroup = %u\n", + (1.0e-6 * (double)(imageW * imageH) / gpuTime), gpuTime, + (imageW * imageH), 1, 0); + + printf("\nReading back GPU results...\n\n"); + checkCudaErrors(cudaMemcpy(h_OutputGPU, d_Output, + imageW * imageH * sizeof(float), + cudaMemcpyDeviceToHost)); + + printf("Checking the results...\n"); + printf(" ...running convolutionRowCPU()\n"); + convolutionRowCPU(h_Buffer, h_Input, h_Kernel, imageW, imageH, KERNEL_RADIUS); + + printf(" ...running convolutionColumnCPU()\n"); + convolutionColumnCPU(h_OutputCPU, h_Buffer, h_Kernel, imageW, imageH, + KERNEL_RADIUS); + + printf(" ...comparing the results\n"); + double sum = 0, delta = 0; + + for (unsigned i = 0; i < imageW * imageH; i++) { + delta += + (h_OutputGPU[i] - h_OutputCPU[i]) * (h_OutputGPU[i] - h_OutputCPU[i]); + sum += h_OutputCPU[i] * h_OutputCPU[i]; + } + + double L2norm = sqrt(delta / sum); + printf(" ...Relative L2 norm: %E\n\n", L2norm); + printf("Shutting down...\n"); + + checkCudaErrors(cudaFree(d_Buffer)); + checkCudaErrors(cudaFree(d_Output)); + checkCudaErrors(cudaFree(d_Input)); + free(h_OutputGPU); + free(h_OutputCPU); + free(h_Buffer); + free(h_Input); + free(h_Kernel); + + sdkDeleteTimer(&hTimer); + + if (L2norm > 1e-6) { + printf("Test failed!\n"); + exit(EXIT_FAILURE); + } + + printf("Test passed\n"); + + sdkStopTimer(&totalTimer); + double totalTime = sdkGetTimerValue(&totalTimer); + printf("Total time: %f ms \n", totalTime); + if (logDir != NULL) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << totalTime << std::endl; + outFile.close(); + } + + sdkDeleteTimer(&totalTimer); + + exit(EXIT_SUCCESS); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/fastWalshTransform.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/fastWalshTransform.cu new file mode 100644 index 0000000..b65f0b1 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/fastWalshTransform.cu @@ -0,0 +1,242 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Walsh transforms belong to a class of generalized Fourier transformations. + * They have applications in various fields of electrical engineering + * and numeric theory. In this sample we demonstrate efficient implementation + * of naturally-ordered Walsh transform + * (also known as Walsh-Hadamard or Hadamard transform) in CUDA and its + * particular application to dyadic convolution computation. + * Refer to excellent Jorg Arndt's "Algorithms for Programmers" textbook + * http://www.jjj.de/fxt/fxtbook.pdf (Chapter 22) + * + * Victor Podlozhnyuk (vpodlozhnyuk@nvidia.com) + */ + +#include +#include +#include +#include +#include +#include + +//////////////////////////////////////////////////////////////////////////////// +// Reference CPU FWT +//////////////////////////////////////////////////////////////////////////////// +void fwtCPU(float *h_Output, float *h_Input, int log2N) { + const int N = 1 << log2N; + + for (int pos = 0; pos < N; pos++) h_Output[pos] = h_Input[pos]; + + // Cycle through stages with different butterfly strides + for (int stride = N / 2; stride >= 1; stride >>= 1) { + // Cycle through subvectors of (2 * stride) elements + for (int base = 0; base < N; base += 2 * stride) + + // Butterfly index within subvector of (2 * stride) size + for (int j = 0; j < stride; j++) { + int i0 = base + j + 0; + int i1 = base + j + stride; + + float T1 = h_Output[i0]; + float T2 = h_Output[i1]; + h_Output[i0] = T1 + T2; + h_Output[i1] = T1 - T2; + } + } +} + +void slowWTcpu(float *h_Output, float *h_Input, int log2N) { + const int N = 1 << log2N; + + for (int i = 0; i < N; i++) { + double sum = 0; + + for (int j = 0; j < N; j++) { + // Walsh-Hadamard quotient + double q = 1.0; + + for (int t = i & j; t != 0; t >>= 1) + if (t & 1) q = -q; + + sum += q * h_Input[j]; + } + + h_Output[i] = (float)sum; + } +} + +void dyadicConvolutionCPU(float *h_Result, float *h_Data, + float *h_Kernel, int log2dataN, + int log2kernelN) { + const int dataN = 1 << log2dataN; + const int kernelN = 1 << log2kernelN; + + for (int i = 0; i < dataN; i++) { + double sum = 0; + + for (int j = 0; j < kernelN; j++) sum += h_Data[i ^ j] * h_Kernel[j]; + + h_Result[i] = (float)sum; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// GPU FWT +//////////////////////////////////////////////////////////////////////////////// +#include "./include/fastWalshTransform_kernel.cuh" + +//////////////////////////////////////////////////////////////////////////////// +// Data configuration +//////////////////////////////////////////////////////////////////////////////// +const int log2Kernel = 7; +const int log2Data = 23; + +const int dataN = 1 << log2Data; +const int kernelN = 1 << log2Kernel; + +const int DATA_SIZE = dataN * sizeof(float); +const int KERNEL_SIZE = kernelN * sizeof(float); + +const double NOPS = 3.0 * (double)dataN * (double)log2Data / 2.0; + +StopWatchInterface *totalTimer = NULL; +//////////////////////////////////////////////////////////////////////////////// +// Main program +//////////////////////////////////////////////////////////////////////////////// +int main(int argc, char *argv[]) { + + char* logDir = NULL; + if (checkCmdLineFlag(argc, (const char **)argv, "logDir")) { + getCmdLineArgumentString(argc, (const char **)argv, "logDir=", &logDir); + } + + sdkCreateTimer(&totalTimer); + sdkStartTimer(&totalTimer); + float *h_Data, *h_Kernel, *h_ResultCPU, *h_ResultGPU; + + float *d_Data, *d_Kernel; + + double delta, ref, sum_delta2, sum_ref2, L2norm, gpuTime; + + StopWatchInterface *hTimer = NULL; + int i; + + printf("%s Starting...\n\n", argv[0]); + + // use command-line specified CUDA device, otherwise use device with highest + // Gflops/s + findCudaDevice(argc, (const char **)argv); + + sdkCreateTimer(&hTimer); + + printf("Initializing data...\n"); + printf("...allocating CPU memory\n"); + h_Kernel = (float *)malloc(KERNEL_SIZE); + h_Data = (float *)malloc(DATA_SIZE); + h_ResultCPU = (float *)malloc(DATA_SIZE); + h_ResultGPU = (float *)malloc(DATA_SIZE); + printf("...allocating GPU memory\n"); + checkCudaErrors(cudaMalloc((void **)&d_Kernel, DATA_SIZE)); + checkCudaErrors(cudaMalloc((void **)&d_Data, DATA_SIZE)); + + printf("...generating data\n"); + printf("Data length: %i; kernel length: %i\n", dataN, kernelN); + srand(2007); + + for (i = 0; i < kernelN; i++) { + h_Kernel[i] = (float)rand() / (float)RAND_MAX; + } + + for (i = 0; i < dataN; i++) { + h_Data[i] = (float)rand() / (float)RAND_MAX; + } + + checkCudaErrors(cudaMemset(d_Kernel, 0, DATA_SIZE)); + checkCudaErrors( + cudaMemcpy(d_Kernel, h_Kernel, KERNEL_SIZE, cudaMemcpyHostToDevice)); + checkCudaErrors( + cudaMemcpy(d_Data, h_Data, DATA_SIZE, cudaMemcpyHostToDevice)); + + printf("Running GPU dyadic convolution using Fast Walsh Transform...\n"); + checkCudaErrors(cudaDeviceSynchronize()); + sdkResetTimer(&hTimer); + sdkStartTimer(&hTimer); + fwtBatchGPU(d_Data, 1, log2Data); + fwtBatchGPU(d_Kernel, 1, log2Data); + modulateGPU(d_Data, d_Kernel, dataN); + fwtBatchGPU(d_Data, 1, log2Data); + checkCudaErrors(cudaDeviceSynchronize()); + sdkStopTimer(&hTimer); + gpuTime = sdkGetTimerValue(&hTimer); + printf("GPU time: %f ms; GOP/s: %f\n", gpuTime, + NOPS / (gpuTime * 0.001 * 1E+9)); + + printf("Reading back GPU results...\n"); + checkCudaErrors( + cudaMemcpy(h_ResultGPU, d_Data, DATA_SIZE, cudaMemcpyDeviceToHost)); + + printf("Running straightforward CPU dyadic convolution...\n"); + dyadicConvolutionCPU(h_ResultCPU, h_Data, h_Kernel, log2Data, log2Kernel); + + printf("Comparing the results...\n"); + sum_delta2 = 0; + sum_ref2 = 0; + + for (i = 0; i < dataN; i++) { + delta = h_ResultCPU[i] - h_ResultGPU[i]; + ref = h_ResultCPU[i]; + sum_delta2 += delta * delta; + sum_ref2 += ref * ref; + } + + L2norm = sqrt(sum_delta2 / sum_ref2); + + printf("Shutting down...\n"); + sdkDeleteTimer(&hTimer); + checkCudaErrors(cudaFree(d_Data)); + checkCudaErrors(cudaFree(d_Kernel)); + free(h_ResultGPU); + free(h_ResultCPU); + free(h_Data); + free(h_Kernel); + + printf("L2 norm: %E\n", L2norm); + printf(L2norm < 1e-6 ? "Test passed\n" : "Test failed!\n"); + + sdkStopTimer(&totalTimer); + double totalTime = sdkGetTimerValue(&totalTimer); + printf("Total time: %f ms \n", totalTime); + if (logDir != NULL) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << totalTime << std::endl; + outFile.close(); + } + + sdkDeleteTimer(&totalTimer); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/BlackScholes_kernel.cuh b/GPU-Virtual-Service/gpu-remoting/src/sample/include/BlackScholes_kernel.cuh new file mode 100644 index 0000000..b7cfccc --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/BlackScholes_kernel.cuh @@ -0,0 +1,167 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +//////////////////////////////////////////////////////////////////////////////// +// Polynomial approximation of cumulative normal distribution function +//////////////////////////////////////////////////////////////////////////////// +static double CND(double d) { + const double A1 = 0.31938153; + const double A2 = -0.356563782; + const double A3 = 1.781477937; + const double A4 = -1.821255978; + const double A5 = 1.330274429; + const double RSQRT2PI = 0.39894228040143267793994605993438; + + double K = 1.0 / (1.0 + 0.2316419 * fabs(d)); + + double cnd = RSQRT2PI * exp(-0.5 * d * d) * + (K * (A1 + K * (A2 + K * (A3 + K * (A4 + K * A5))))); + + if (d > 0) cnd = 1.0 - cnd; + + return cnd; +} + +//////////////////////////////////////////////////////////////////////////////// +// Black-Scholes formula for both call and put +//////////////////////////////////////////////////////////////////////////////// +static void BlackScholesBodyCPU(float &callResult, float &putResult, + float Sf, // Stock price + float Xf, // Option strike + float Tf, // Option years + float Rf, // Riskless rate + float Vf // Volatility rate + ) { + double S = Sf, X = Xf, T = Tf, R = Rf, V = Vf; + + double sqrtT = sqrt(T); + double d1 = (log(S / X) + (R + 0.5 * V * V) * T) / (V * sqrtT); + double d2 = d1 - V * sqrtT; + double CNDD1 = CND(d1); + double CNDD2 = CND(d2); + + // Calculate Call and Put simultaneously + double expRT = exp(-R * T); + callResult = (float)(S * CNDD1 - X * expRT * CNDD2); + putResult = (float)(X * expRT * (1.0 - CNDD2) - S * (1.0 - CNDD1)); +} + +//////////////////////////////////////////////////////////////////////////////// +// Process an array of optN options +//////////////////////////////////////////////////////////////////////////////// +extern "C" void BlackScholesCPU(float *h_CallResult, float *h_PutResult, + float *h_StockPrice, float *h_OptionStrike, + float *h_OptionYears, float Riskfree, + float Volatility, int optN) { + for (int opt = 0; opt < optN; opt++) + BlackScholesBodyCPU(h_CallResult[opt], h_PutResult[opt], h_StockPrice[opt], + h_OptionStrike[opt], h_OptionYears[opt], Riskfree, + Volatility); +} + + +//////////////////////////////////////////////////////////////////////////////// +// Polynomial approximation of cumulative normal distribution function +//////////////////////////////////////////////////////////////////////////////// +__device__ inline float cndGPU(float d) { + const float A1 = 0.31938153f; + const float A2 = -0.356563782f; + const float A3 = 1.781477937f; + const float A4 = -1.821255978f; + const float A5 = 1.330274429f; + const float RSQRT2PI = 0.39894228040143267793994605993438f; + + float K = __fdividef(1.0f, (1.0f + 0.2316419f * fabsf(d))); + + float cnd = RSQRT2PI * __expf(-0.5f * d * d) * + (K * (A1 + K * (A2 + K * (A3 + K * (A4 + K * A5))))); + + if (d > 0) cnd = 1.0f - cnd; + + return cnd; +} + +//////////////////////////////////////////////////////////////////////////////// +// Black-Scholes formula for both call and put +//////////////////////////////////////////////////////////////////////////////// +__device__ inline void BlackScholesBodyGPU(float &CallResult, float &PutResult, + float S, // Stock price + float X, // Option strike + float T, // Option years + float R, // Riskless rate + float V // Volatility rate + ) { + float sqrtT, expRT; + float d1, d2, CNDD1, CNDD2; + + sqrtT = __fdividef(1.0F, rsqrtf(T)); + d1 = __fdividef(__logf(S / X) + (R + 0.5f * V * V) * T, V * sqrtT); + d2 = d1 - V * sqrtT; + + CNDD1 = cndGPU(d1); + CNDD2 = cndGPU(d2); + + // Calculate Call and Put simultaneously + expRT = __expf(-R * T); + CallResult = S * CNDD1 - X * expRT * CNDD2; + PutResult = X * expRT * (1.0f - CNDD2) - S * (1.0f - CNDD1); +} + +//////////////////////////////////////////////////////////////////////////////// +// Process an array of optN options on GPU +//////////////////////////////////////////////////////////////////////////////// +__launch_bounds__(128) __global__ + void BlackScholesGPU(float2 *__restrict d_CallResult, + float2 *__restrict d_PutResult, + float2 *__restrict d_StockPrice, + float2 *__restrict d_OptionStrike, + float2 *__restrict d_OptionYears, float Riskfree, + float Volatility, int optN) { + ////Thread index + // const int tid = blockDim.x * blockIdx.x + threadIdx.x; + ////Total number of threads in execution grid + // const int THREAD_N = blockDim.x * gridDim.x; + + const int opt = blockDim.x * blockIdx.x + threadIdx.x; + + // Calculating 2 options per thread to increase ILP (instruction level + // parallelism) + if (opt < (optN / 2)) { + float callResult1, callResult2; + float putResult1, putResult2; + BlackScholesBodyGPU(callResult1, putResult1, d_StockPrice[opt].x, + d_OptionStrike[opt].x, d_OptionYears[opt].x, Riskfree, + Volatility); + BlackScholesBodyGPU(callResult2, putResult2, d_StockPrice[opt].y, + d_OptionStrike[opt].y, d_OptionYears[opt].y, Riskfree, + Volatility); + d_CallResult[opt] = make_float2(callResult1, callResult2); + d_PutResult[opt] = make_float2(putResult1, putResult2); + } +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/convolutionSeparable_common.cuh b/GPU-Virtual-Service/gpu-remoting/src/sample/include/convolutionSeparable_common.cuh new file mode 100644 index 0000000..f968521 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/convolutionSeparable_common.cuh @@ -0,0 +1,268 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef CONVOLUTIONSEPARABLE_COMMON_H +#define CONVOLUTIONSEPARABLE_COMMON_H + +#include +#include +#include + +namespace cg = cooperative_groups; + +#define KERNEL_RADIUS 8 +#define KERNEL_LENGTH (2 * KERNEL_RADIUS + 1) + + + +//////////////////////////////////////////////////////////////////////////////// +// Reference row convolution filter +//////////////////////////////////////////////////////////////////////////////// +void convolutionRowCPU(float *h_Dst, float *h_Src, float *h_Kernel, + int imageW, int imageH, int kernelR) { + for (int y = 0; y < imageH; y++) + for (int x = 0; x < imageW; x++) { + float sum = 0; + + for (int k = -kernelR; k <= kernelR; k++) { + int d = x + k; + + if (d >= 0 && d < imageW) + sum += h_Src[y * imageW + d] * h_Kernel[kernelR - k]; + } + + h_Dst[y * imageW + x] = sum; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Reference column convolution filter +//////////////////////////////////////////////////////////////////////////////// +void convolutionColumnCPU(float *h_Dst, float *h_Src, + float *h_Kernel, int imageW, int imageH, + int kernelR) { + for (int y = 0; y < imageH; y++) + for (int x = 0; x < imageW; x++) { + float sum = 0; + + for (int k = -kernelR; k <= kernelR; k++) { + int d = y + k; + + if (d >= 0 && d < imageH) + sum += h_Src[d * imageW + x] * h_Kernel[kernelR - k]; + } + + h_Dst[y * imageW + x] = sum; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// GPU convolution +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +// Convolution kernel storage +//////////////////////////////////////////////////////////////////////////////// +__constant__ float c_Kernel[KERNEL_LENGTH]; + +void setConvolutionKernel(float *h_Kernel) { + cudaMemcpyToSymbol(c_Kernel, h_Kernel, KERNEL_LENGTH * sizeof(float)); +} + +//////////////////////////////////////////////////////////////////////////////// +// Row convolution filter +//////////////////////////////////////////////////////////////////////////////// +#define ROWS_BLOCKDIM_X 16 +#define ROWS_BLOCKDIM_Y 4 +#define ROWS_RESULT_STEPS 8 +#define ROWS_HALO_STEPS 1 + +__global__ void convolutionRowsKernel(float *d_Dst, float *d_Src, int imageW, + int imageH, int pitch) { + // Handle to thread block group + cg::thread_block cta = cg::this_thread_block(); + __shared__ float + s_Data[ROWS_BLOCKDIM_Y][(ROWS_RESULT_STEPS + 2 * ROWS_HALO_STEPS) * + ROWS_BLOCKDIM_X]; + + // Offset to the left halo edge + const int baseX = + (blockIdx.x * ROWS_RESULT_STEPS - ROWS_HALO_STEPS) * ROWS_BLOCKDIM_X + + threadIdx.x; + const int baseY = blockIdx.y * ROWS_BLOCKDIM_Y + threadIdx.y; + + d_Src += baseY * pitch + baseX; + d_Dst += baseY * pitch + baseX; + +// Load main data +#pragma unroll + + for (int i = ROWS_HALO_STEPS; i < ROWS_HALO_STEPS + ROWS_RESULT_STEPS; i++) { + s_Data[threadIdx.y][threadIdx.x + i * ROWS_BLOCKDIM_X] = + d_Src[i * ROWS_BLOCKDIM_X]; + } + +// Load left halo +#pragma unroll + + for (int i = 0; i < ROWS_HALO_STEPS; i++) { + s_Data[threadIdx.y][threadIdx.x + i * ROWS_BLOCKDIM_X] = + (baseX >= -i * ROWS_BLOCKDIM_X) ? d_Src[i * ROWS_BLOCKDIM_X] : 0; + } + +// Load right halo +#pragma unroll + + for (int i = ROWS_HALO_STEPS + ROWS_RESULT_STEPS; + i < ROWS_HALO_STEPS + ROWS_RESULT_STEPS + ROWS_HALO_STEPS; i++) { + s_Data[threadIdx.y][threadIdx.x + i * ROWS_BLOCKDIM_X] = + (imageW - baseX > i * ROWS_BLOCKDIM_X) ? d_Src[i * ROWS_BLOCKDIM_X] : 0; + } + + // Compute and store results + cg::sync(cta); +#pragma unroll + + for (int i = ROWS_HALO_STEPS; i < ROWS_HALO_STEPS + ROWS_RESULT_STEPS; i++) { + float sum = 0; + +#pragma unroll + + for (int j = -KERNEL_RADIUS; j <= KERNEL_RADIUS; j++) { + sum += c_Kernel[KERNEL_RADIUS - j] * + s_Data[threadIdx.y][threadIdx.x + i * ROWS_BLOCKDIM_X + j]; + } + + d_Dst[i * ROWS_BLOCKDIM_X] = sum; + } +} + +void convolutionRowsGPU(float *d_Dst, float *d_Src, int imageW, + int imageH) { + assert(ROWS_BLOCKDIM_X * ROWS_HALO_STEPS >= KERNEL_RADIUS); + assert(imageW % (ROWS_RESULT_STEPS * ROWS_BLOCKDIM_X) == 0); + assert(imageH % ROWS_BLOCKDIM_Y == 0); + + dim3 blocks(imageW / (ROWS_RESULT_STEPS * ROWS_BLOCKDIM_X), + imageH / ROWS_BLOCKDIM_Y); + dim3 threads(ROWS_BLOCKDIM_X, ROWS_BLOCKDIM_Y); + + convolutionRowsKernel<<>>(d_Dst, d_Src, imageW, imageH, + imageW); + getLastCudaError("convolutionRowsKernel() execution failed\n"); +} + +//////////////////////////////////////////////////////////////////////////////// +// Column convolution filter +//////////////////////////////////////////////////////////////////////////////// +#define COLUMNS_BLOCKDIM_X 16 +#define COLUMNS_BLOCKDIM_Y 8 +#define COLUMNS_RESULT_STEPS 8 +#define COLUMNS_HALO_STEPS 1 + +__global__ void convolutionColumnsKernel(float *d_Dst, float *d_Src, int imageW, + int imageH, int pitch) { + // Handle to thread block group + cg::thread_block cta = cg::this_thread_block(); + __shared__ float s_Data[COLUMNS_BLOCKDIM_X][(COLUMNS_RESULT_STEPS + + 2 * COLUMNS_HALO_STEPS) * + COLUMNS_BLOCKDIM_Y + + 1]; + + // Offset to the upper halo edge + const int baseX = blockIdx.x * COLUMNS_BLOCKDIM_X + threadIdx.x; + const int baseY = (blockIdx.y * COLUMNS_RESULT_STEPS - COLUMNS_HALO_STEPS) * + COLUMNS_BLOCKDIM_Y + + threadIdx.y; + d_Src += baseY * pitch + baseX; + d_Dst += baseY * pitch + baseX; + +// Main data +#pragma unroll + + for (int i = COLUMNS_HALO_STEPS; + i < COLUMNS_HALO_STEPS + COLUMNS_RESULT_STEPS; i++) { + s_Data[threadIdx.x][threadIdx.y + i * COLUMNS_BLOCKDIM_Y] = + d_Src[i * COLUMNS_BLOCKDIM_Y * pitch]; + } + +// Upper halo +#pragma unroll + + for (int i = 0; i < COLUMNS_HALO_STEPS; i++) { + s_Data[threadIdx.x][threadIdx.y + i * COLUMNS_BLOCKDIM_Y] = + (baseY >= -i * COLUMNS_BLOCKDIM_Y) + ? d_Src[i * COLUMNS_BLOCKDIM_Y * pitch] + : 0; + } + +// Lower halo +#pragma unroll + + for (int i = COLUMNS_HALO_STEPS + COLUMNS_RESULT_STEPS; + i < COLUMNS_HALO_STEPS + COLUMNS_RESULT_STEPS + COLUMNS_HALO_STEPS; + i++) { + s_Data[threadIdx.x][threadIdx.y + i * COLUMNS_BLOCKDIM_Y] = + (imageH - baseY > i * COLUMNS_BLOCKDIM_Y) + ? d_Src[i * COLUMNS_BLOCKDIM_Y * pitch] + : 0; + } + + // Compute and store results + cg::sync(cta); +#pragma unroll + + for (int i = COLUMNS_HALO_STEPS; + i < COLUMNS_HALO_STEPS + COLUMNS_RESULT_STEPS; i++) { + float sum = 0; +#pragma unroll + + for (int j = -KERNEL_RADIUS; j <= KERNEL_RADIUS; j++) { + sum += c_Kernel[KERNEL_RADIUS - j] * + s_Data[threadIdx.x][threadIdx.y + i * COLUMNS_BLOCKDIM_Y + j]; + } + + d_Dst[i * COLUMNS_BLOCKDIM_Y * pitch] = sum; + } +} + +void convolutionColumnsGPU(float *d_Dst, float *d_Src, int imageW, + int imageH) { + assert(COLUMNS_BLOCKDIM_Y * COLUMNS_HALO_STEPS >= KERNEL_RADIUS); + assert(imageW % COLUMNS_BLOCKDIM_X == 0); + assert(imageH % (COLUMNS_RESULT_STEPS * COLUMNS_BLOCKDIM_Y) == 0); + + dim3 blocks(imageW / COLUMNS_BLOCKDIM_X, + imageH / (COLUMNS_RESULT_STEPS * COLUMNS_BLOCKDIM_Y)); + dim3 threads(COLUMNS_BLOCKDIM_X, COLUMNS_BLOCKDIM_Y); + + convolutionColumnsKernel<<>>(d_Dst, d_Src, imageW, imageH, + imageW); + getLastCudaError("convolutionColumnsKernel() execution failed\n"); +} + +#endif diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/exception.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/exception.h new file mode 100644 index 0000000..ca8ac25 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/exception.h @@ -0,0 +1,151 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* CUda UTility Library */ +#ifndef COMMON_EXCEPTION_H_ +#define COMMON_EXCEPTION_H_ + +// includes, system +#include +#include +#include +#include +#include + +//! Exception wrapper. +//! @param Std_Exception Exception out of namespace std for easy typing. +template +class Exception : public Std_Exception { + public: + //! @brief Static construction interface + //! @return Alwayss throws ( Located_Exception) + //! @param file file in which the Exception occurs + //! @param line line in which the Exception occurs + //! @param detailed details on the code fragment causing the Exception + static void throw_it(const char *file, const int line, + const char *detailed = "-"); + + //! Static construction interface + //! @return Alwayss throws ( Located_Exception) + //! @param file file in which the Exception occurs + //! @param line line in which the Exception occurs + //! @param detailed details on the code fragment causing the Exception + static void throw_it(const char *file, const int line, + const std::string &detailed); + + //! Destructor + virtual ~Exception() throw(); + + private: + //! Constructor, default (private) + Exception(); + + //! Constructor, standard + //! @param str string returned by what() + explicit Exception(const std::string &str); +}; + +//////////////////////////////////////////////////////////////////////////////// +//! Exception handler function for arbitrary exceptions +//! @param ex exception to handle +//////////////////////////////////////////////////////////////////////////////// +template +inline void handleException(const Exception_Typ &ex) { + std::cerr << ex.what() << std::endl; + + exit(EXIT_FAILURE); +} + +//! Convenience macros + +//! Exception caused by dynamic program behavior, e.g. file does not exist +#define RUNTIME_EXCEPTION(msg) \ + Exception::throw_it(__FILE__, __LINE__, msg) + +//! Logic exception in program, e.g. an assert failed +#define LOGIC_EXCEPTION(msg) \ + Exception::throw_it(__FILE__, __LINE__, msg) + +//! Out of range exception +#define RANGE_EXCEPTION(msg) \ + Exception::throw_it(__FILE__, __LINE__, msg) + +//////////////////////////////////////////////////////////////////////////////// +//! Implementation + +// includes, system +#include + +//////////////////////////////////////////////////////////////////////////////// +//! Static construction interface. +//! @param Exception causing code fragment (file and line) and detailed infos. +//////////////////////////////////////////////////////////////////////////////// +/*static*/ template +void Exception::throw_it(const char *file, const int line, + const char *detailed) { + std::stringstream s; + + // Quiet heavy-weight but exceptions are not for + // performance / release versions + s << "Exception in file '" << file << "' in line " << line << "\n" + << "Detailed description: " << detailed << "\n"; + + throw Exception(s.str()); +} + +//////////////////////////////////////////////////////////////////////////////// +//! Static construction interface. +//! @param Exception causing code fragment (file and line) and detailed infos. +//////////////////////////////////////////////////////////////////////////////// +/*static*/ template +void Exception::throw_it(const char *file, const int line, + const std::string &msg) { + throw_it(file, line, msg.c_str()); +} + +//////////////////////////////////////////////////////////////////////////////// +//! Constructor, default (private). +//////////////////////////////////////////////////////////////////////////////// +template +Exception::Exception() : Std_Exception("Unknown Exception.\n") {} + +//////////////////////////////////////////////////////////////////////////////// +//! Constructor, standard (private). +//! String returned by what(). +//////////////////////////////////////////////////////////////////////////////// +template +Exception::Exception(const std::string &s) : Std_Exception(s) {} + +//////////////////////////////////////////////////////////////////////////////// +//! Destructor +//////////////////////////////////////////////////////////////////////////////// +template +Exception::~Exception() throw() {} + + // functions, exported + +#endif // COMMON_EXCEPTION_H_ diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/fastWalshTransform_kernel.cuh b/GPU-Virtual-Service/gpu-remoting/src/sample/include/fastWalshTransform_kernel.cuh new file mode 100644 index 0000000..4f64117 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/fastWalshTransform_kernel.cuh @@ -0,0 +1,185 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef FWT_KERNEL_CUH +#define FWT_KERNEL_CUH +#ifndef fwt_kernel_cuh +#define fwt_kernel_cuh + +#include + +namespace cg = cooperative_groups; + +/////////////////////////////////////////////////////////////////////////////// +// Elementary(for vectors less than elementary size) in-shared memory +// combined radix-2 + radix-4 Fast Walsh Transform +/////////////////////////////////////////////////////////////////////////////// +#define ELEMENTARY_LOG2SIZE 11 + +__global__ void fwtBatch1Kernel(float *d_Output, float *d_Input, int log2N) { + // Handle to thread block group + cg::thread_block cta = cg::this_thread_block(); + const int N = 1 << log2N; + const int base = blockIdx.x << log2N; + + //(2 ** 11) * 4 bytes == 8KB -- maximum s_data[] size for G80 + extern __shared__ float s_data[]; + float *d_Src = d_Input + base; + float *d_Dst = d_Output + base; + + for (int pos = threadIdx.x; pos < N; pos += blockDim.x) { + s_data[pos] = d_Src[pos]; + } + + // Main radix-4 stages + const int pos = threadIdx.x; + + for (int stride = N >> 2; stride > 0; stride >>= 2) { + int lo = pos & (stride - 1); + int i0 = ((pos - lo) << 2) + lo; + int i1 = i0 + stride; + int i2 = i1 + stride; + int i3 = i2 + stride; + + cg::sync(cta); + float D0 = s_data[i0]; + float D1 = s_data[i1]; + float D2 = s_data[i2]; + float D3 = s_data[i3]; + + float T; + T = D0; + D0 = D0 + D2; + D2 = T - D2; + T = D1; + D1 = D1 + D3; + D3 = T - D3; + T = D0; + s_data[i0] = D0 + D1; + s_data[i1] = T - D1; + T = D2; + s_data[i2] = D2 + D3; + s_data[i3] = T - D3; + } + + // Do single radix-2 stage for odd power of two + if (log2N & 1) { + cg::sync(cta); + + for (int pos = threadIdx.x; pos < N / 2; pos += blockDim.x) { + int i0 = pos << 1; + int i1 = i0 + 1; + + float D0 = s_data[i0]; + float D1 = s_data[i1]; + s_data[i0] = D0 + D1; + s_data[i1] = D0 - D1; + } + } + + cg::sync(cta); + + for (int pos = threadIdx.x; pos < N; pos += blockDim.x) { + d_Dst[pos] = s_data[pos]; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Single in-global memory radix-4 Fast Walsh Transform pass +// (for strides exceeding elementary vector size) +//////////////////////////////////////////////////////////////////////////////// +__global__ void fwtBatch2Kernel(float *d_Output, float *d_Input, int stride) { + const int pos = blockIdx.x * blockDim.x + threadIdx.x; + const int N = blockDim.x * gridDim.x * 4; + + float *d_Src = d_Input + blockIdx.y * N; + float *d_Dst = d_Output + blockIdx.y * N; + + int lo = pos & (stride - 1); + int i0 = ((pos - lo) << 2) + lo; + int i1 = i0 + stride; + int i2 = i1 + stride; + int i3 = i2 + stride; + + float D0 = d_Src[i0]; + float D1 = d_Src[i1]; + float D2 = d_Src[i2]; + float D3 = d_Src[i3]; + + float T; + T = D0; + D0 = D0 + D2; + D2 = T - D2; + T = D1; + D1 = D1 + D3; + D3 = T - D3; + T = D0; + d_Dst[i0] = D0 + D1; + d_Dst[i1] = T - D1; + T = D2; + d_Dst[i2] = D2 + D3; + d_Dst[i3] = T - D3; +} + +//////////////////////////////////////////////////////////////////////////////// +// Put everything together: batched Fast Walsh Transform CPU front-end +//////////////////////////////////////////////////////////////////////////////// +void fwtBatchGPU(float *d_Data, int M, int log2N) { + const int THREAD_N = 256; + + int N = 1 << log2N; + dim3 grid((1 << log2N) / (4 * THREAD_N), M, 1); + + for (; log2N > ELEMENTARY_LOG2SIZE; log2N -= 2, N >>= 2, M <<= 2) { + fwtBatch2Kernel<<>>(d_Data, d_Data, N / 4); + getLastCudaError("fwtBatch2Kernel() execution failed\n"); + } + + fwtBatch1Kernel<<>>(d_Data, d_Data, log2N); + getLastCudaError("fwtBatch1Kernel() execution failed\n"); +} + +//////////////////////////////////////////////////////////////////////////////// +// Modulate two arrays +//////////////////////////////////////////////////////////////////////////////// +__global__ void modulateKernel(float *d_A, float *d_B, int N) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + int numThreads = blockDim.x * gridDim.x; + float rcpN = 1.0f / (float)N; + + for (int pos = tid; pos < N; pos += numThreads) { + d_A[pos] *= d_B[pos] * rcpN; + } +} + +// Interface to modulateKernel() +void modulateGPU(float *d_A, float *d_B, int N) { + modulateKernel<<<128, 256>>>(d_A, d_B, N); +} + +#endif +#endif diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_cuda.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_cuda.h new file mode 100644 index 0000000..441b88d --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_cuda.h @@ -0,0 +1,976 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +//////////////////////////////////////////////////////////////////////////////// +// These are CUDA Helper functions for initialization and error checking + +#ifndef COMMON_HELPER_CUDA_H_ +#define COMMON_HELPER_CUDA_H_ + +#pragma once + +#include +#include +#include +#include + +#include + +#ifndef EXIT_WAIVED +#define EXIT_WAIVED 2 +#endif + +// Note, it is required that your SDK sample to include the proper header +// files, please refer the CUDA examples for examples of the needed CUDA +// headers, which may change depending on which CUDA functions are used. + +// CUDA Runtime error messages +#ifdef __DRIVER_TYPES_H__ +static const char *_cudaGetErrorEnum(cudaError_t error) { + return cudaGetErrorName(error); +} +#endif + +#ifdef CUDA_DRIVER_API +// CUDA Driver API errors +static const char *_cudaGetErrorEnum(CUresult error) { + static char unknown[] = ""; + const char *ret = NULL; + cuGetErrorName(error, &ret); + return ret ? ret : unknown; +} +#endif + +#ifdef CUBLAS_API_H_ +// cuBLAS API errors +static const char *_cudaGetErrorEnum(cublasStatus_t error) { + switch (error) { + case CUBLAS_STATUS_SUCCESS: + return "CUBLAS_STATUS_SUCCESS"; + + case CUBLAS_STATUS_NOT_INITIALIZED: + return "CUBLAS_STATUS_NOT_INITIALIZED"; + + case CUBLAS_STATUS_ALLOC_FAILED: + return "CUBLAS_STATUS_ALLOC_FAILED"; + + case CUBLAS_STATUS_INVALID_VALUE: + return "CUBLAS_STATUS_INVALID_VALUE"; + + case CUBLAS_STATUS_ARCH_MISMATCH: + return "CUBLAS_STATUS_ARCH_MISMATCH"; + + case CUBLAS_STATUS_MAPPING_ERROR: + return "CUBLAS_STATUS_MAPPING_ERROR"; + + case CUBLAS_STATUS_EXECUTION_FAILED: + return "CUBLAS_STATUS_EXECUTION_FAILED"; + + case CUBLAS_STATUS_INTERNAL_ERROR: + return "CUBLAS_STATUS_INTERNAL_ERROR"; + + case CUBLAS_STATUS_NOT_SUPPORTED: + return "CUBLAS_STATUS_NOT_SUPPORTED"; + + case CUBLAS_STATUS_LICENSE_ERROR: + return "CUBLAS_STATUS_LICENSE_ERROR"; + } + + return ""; +} +#endif + +#ifdef _CUFFT_H_ +// cuFFT API errors +static const char *_cudaGetErrorEnum(cufftResult error) { + switch (error) { + case CUFFT_SUCCESS: + return "CUFFT_SUCCESS"; + + case CUFFT_INVALID_PLAN: + return "CUFFT_INVALID_PLAN"; + + case CUFFT_ALLOC_FAILED: + return "CUFFT_ALLOC_FAILED"; + + case CUFFT_INVALID_TYPE: + return "CUFFT_INVALID_TYPE"; + + case CUFFT_INVALID_VALUE: + return "CUFFT_INVALID_VALUE"; + + case CUFFT_INTERNAL_ERROR: + return "CUFFT_INTERNAL_ERROR"; + + case CUFFT_EXEC_FAILED: + return "CUFFT_EXEC_FAILED"; + + case CUFFT_SETUP_FAILED: + return "CUFFT_SETUP_FAILED"; + + case CUFFT_INVALID_SIZE: + return "CUFFT_INVALID_SIZE"; + + case CUFFT_UNALIGNED_DATA: + return "CUFFT_UNALIGNED_DATA"; + + case CUFFT_INCOMPLETE_PARAMETER_LIST: + return "CUFFT_INCOMPLETE_PARAMETER_LIST"; + + case CUFFT_INVALID_DEVICE: + return "CUFFT_INVALID_DEVICE"; + + case CUFFT_PARSE_ERROR: + return "CUFFT_PARSE_ERROR"; + + case CUFFT_NO_WORKSPACE: + return "CUFFT_NO_WORKSPACE"; + + case CUFFT_NOT_IMPLEMENTED: + return "CUFFT_NOT_IMPLEMENTED"; + + case CUFFT_LICENSE_ERROR: + return "CUFFT_LICENSE_ERROR"; + + case CUFFT_NOT_SUPPORTED: + return "CUFFT_NOT_SUPPORTED"; + } + + return ""; +} +#endif + +#ifdef CUSPARSEAPI +// cuSPARSE API errors +static const char *_cudaGetErrorEnum(cusparseStatus_t error) { + switch (error) { + case CUSPARSE_STATUS_SUCCESS: + return "CUSPARSE_STATUS_SUCCESS"; + + case CUSPARSE_STATUS_NOT_INITIALIZED: + return "CUSPARSE_STATUS_NOT_INITIALIZED"; + + case CUSPARSE_STATUS_ALLOC_FAILED: + return "CUSPARSE_STATUS_ALLOC_FAILED"; + + case CUSPARSE_STATUS_INVALID_VALUE: + return "CUSPARSE_STATUS_INVALID_VALUE"; + + case CUSPARSE_STATUS_ARCH_MISMATCH: + return "CUSPARSE_STATUS_ARCH_MISMATCH"; + + case CUSPARSE_STATUS_MAPPING_ERROR: + return "CUSPARSE_STATUS_MAPPING_ERROR"; + + case CUSPARSE_STATUS_EXECUTION_FAILED: + return "CUSPARSE_STATUS_EXECUTION_FAILED"; + + case CUSPARSE_STATUS_INTERNAL_ERROR: + return "CUSPARSE_STATUS_INTERNAL_ERROR"; + + case CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED: + return "CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED"; + } + + return ""; +} +#endif + +#ifdef CUSOLVER_COMMON_H_ +// cuSOLVER API errors +static const char *_cudaGetErrorEnum(cusolverStatus_t error) { + switch (error) { + case CUSOLVER_STATUS_SUCCESS: + return "CUSOLVER_STATUS_SUCCESS"; + case CUSOLVER_STATUS_NOT_INITIALIZED: + return "CUSOLVER_STATUS_NOT_INITIALIZED"; + case CUSOLVER_STATUS_ALLOC_FAILED: + return "CUSOLVER_STATUS_ALLOC_FAILED"; + case CUSOLVER_STATUS_INVALID_VALUE: + return "CUSOLVER_STATUS_INVALID_VALUE"; + case CUSOLVER_STATUS_ARCH_MISMATCH: + return "CUSOLVER_STATUS_ARCH_MISMATCH"; + case CUSOLVER_STATUS_MAPPING_ERROR: + return "CUSOLVER_STATUS_MAPPING_ERROR"; + case CUSOLVER_STATUS_EXECUTION_FAILED: + return "CUSOLVER_STATUS_EXECUTION_FAILED"; + case CUSOLVER_STATUS_INTERNAL_ERROR: + return "CUSOLVER_STATUS_INTERNAL_ERROR"; + case CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED: + return "CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED"; + case CUSOLVER_STATUS_NOT_SUPPORTED: + return "CUSOLVER_STATUS_NOT_SUPPORTED "; + case CUSOLVER_STATUS_ZERO_PIVOT: + return "CUSOLVER_STATUS_ZERO_PIVOT"; + case CUSOLVER_STATUS_INVALID_LICENSE: + return "CUSOLVER_STATUS_INVALID_LICENSE"; + } + + return ""; +} +#endif + +#ifdef CURAND_H_ +// cuRAND API errors +static const char *_cudaGetErrorEnum(curandStatus_t error) { + switch (error) { + case CURAND_STATUS_SUCCESS: + return "CURAND_STATUS_SUCCESS"; + + case CURAND_STATUS_VERSION_MISMATCH: + return "CURAND_STATUS_VERSION_MISMATCH"; + + case CURAND_STATUS_NOT_INITIALIZED: + return "CURAND_STATUS_NOT_INITIALIZED"; + + case CURAND_STATUS_ALLOCATION_FAILED: + return "CURAND_STATUS_ALLOCATION_FAILED"; + + case CURAND_STATUS_TYPE_ERROR: + return "CURAND_STATUS_TYPE_ERROR"; + + case CURAND_STATUS_OUT_OF_RANGE: + return "CURAND_STATUS_OUT_OF_RANGE"; + + case CURAND_STATUS_LENGTH_NOT_MULTIPLE: + return "CURAND_STATUS_LENGTH_NOT_MULTIPLE"; + + case CURAND_STATUS_DOUBLE_PRECISION_REQUIRED: + return "CURAND_STATUS_DOUBLE_PRECISION_REQUIRED"; + + case CURAND_STATUS_LAUNCH_FAILURE: + return "CURAND_STATUS_LAUNCH_FAILURE"; + + case CURAND_STATUS_PREEXISTING_FAILURE: + return "CURAND_STATUS_PREEXISTING_FAILURE"; + + case CURAND_STATUS_INITIALIZATION_FAILED: + return "CURAND_STATUS_INITIALIZATION_FAILED"; + + case CURAND_STATUS_ARCH_MISMATCH: + return "CURAND_STATUS_ARCH_MISMATCH"; + + case CURAND_STATUS_INTERNAL_ERROR: + return "CURAND_STATUS_INTERNAL_ERROR"; + } + + return ""; +} +#endif + +#ifdef NVJPEGAPI +// nvJPEG API errors +static const char *_cudaGetErrorEnum(nvjpegStatus_t error) { + switch (error) { + case NVJPEG_STATUS_SUCCESS: + return "NVJPEG_STATUS_SUCCESS"; + + case NVJPEG_STATUS_NOT_INITIALIZED: + return "NVJPEG_STATUS_NOT_INITIALIZED"; + + case NVJPEG_STATUS_INVALID_PARAMETER: + return "NVJPEG_STATUS_INVALID_PARAMETER"; + + case NVJPEG_STATUS_BAD_JPEG: + return "NVJPEG_STATUS_BAD_JPEG"; + + case NVJPEG_STATUS_JPEG_NOT_SUPPORTED: + return "NVJPEG_STATUS_JPEG_NOT_SUPPORTED"; + + case NVJPEG_STATUS_ALLOCATOR_FAILURE: + return "NVJPEG_STATUS_ALLOCATOR_FAILURE"; + + case NVJPEG_STATUS_EXECUTION_FAILED: + return "NVJPEG_STATUS_EXECUTION_FAILED"; + + case NVJPEG_STATUS_ARCH_MISMATCH: + return "NVJPEG_STATUS_ARCH_MISMATCH"; + + case NVJPEG_STATUS_INTERNAL_ERROR: + return "NVJPEG_STATUS_INTERNAL_ERROR"; + } + + return ""; +} +#endif + +#ifdef NV_NPPIDEFS_H +// NPP API errors +static const char *_cudaGetErrorEnum(NppStatus error) { + switch (error) { + case NPP_NOT_SUPPORTED_MODE_ERROR: + return "NPP_NOT_SUPPORTED_MODE_ERROR"; + + case NPP_ROUND_MODE_NOT_SUPPORTED_ERROR: + return "NPP_ROUND_MODE_NOT_SUPPORTED_ERROR"; + + case NPP_RESIZE_NO_OPERATION_ERROR: + return "NPP_RESIZE_NO_OPERATION_ERROR"; + + case NPP_NOT_SUFFICIENT_COMPUTE_CAPABILITY: + return "NPP_NOT_SUFFICIENT_COMPUTE_CAPABILITY"; + +#if ((NPP_VERSION_MAJOR << 12) + (NPP_VERSION_MINOR << 4)) <= 0x5000 + + case NPP_BAD_ARG_ERROR: + return "NPP_BAD_ARGUMENT_ERROR"; + + case NPP_COEFF_ERROR: + return "NPP_COEFFICIENT_ERROR"; + + case NPP_RECT_ERROR: + return "NPP_RECTANGLE_ERROR"; + + case NPP_QUAD_ERROR: + return "NPP_QUADRANGLE_ERROR"; + + case NPP_MEM_ALLOC_ERR: + return "NPP_MEMORY_ALLOCATION_ERROR"; + + case NPP_HISTO_NUMBER_OF_LEVELS_ERROR: + return "NPP_HISTOGRAM_NUMBER_OF_LEVELS_ERROR"; + + case NPP_INVALID_INPUT: + return "NPP_INVALID_INPUT"; + + case NPP_POINTER_ERROR: + return "NPP_POINTER_ERROR"; + + case NPP_WARNING: + return "NPP_WARNING"; + + case NPP_ODD_ROI_WARNING: + return "NPP_ODD_ROI_WARNING"; +#else + + // These are for CUDA 5.5 or higher + case NPP_BAD_ARGUMENT_ERROR: + return "NPP_BAD_ARGUMENT_ERROR"; + + case NPP_COEFFICIENT_ERROR: + return "NPP_COEFFICIENT_ERROR"; + + case NPP_RECTANGLE_ERROR: + return "NPP_RECTANGLE_ERROR"; + + case NPP_QUADRANGLE_ERROR: + return "NPP_QUADRANGLE_ERROR"; + + case NPP_MEMORY_ALLOCATION_ERR: + return "NPP_MEMORY_ALLOCATION_ERROR"; + + case NPP_HISTOGRAM_NUMBER_OF_LEVELS_ERROR: + return "NPP_HISTOGRAM_NUMBER_OF_LEVELS_ERROR"; + + case NPP_INVALID_HOST_POINTER_ERROR: + return "NPP_INVALID_HOST_POINTER_ERROR"; + + case NPP_INVALID_DEVICE_POINTER_ERROR: + return "NPP_INVALID_DEVICE_POINTER_ERROR"; +#endif + + case NPP_LUT_NUMBER_OF_LEVELS_ERROR: + return "NPP_LUT_NUMBER_OF_LEVELS_ERROR"; + + case NPP_TEXTURE_BIND_ERROR: + return "NPP_TEXTURE_BIND_ERROR"; + + case NPP_WRONG_INTERSECTION_ROI_ERROR: + return "NPP_WRONG_INTERSECTION_ROI_ERROR"; + + case NPP_NOT_EVEN_STEP_ERROR: + return "NPP_NOT_EVEN_STEP_ERROR"; + + case NPP_INTERPOLATION_ERROR: + return "NPP_INTERPOLATION_ERROR"; + + case NPP_RESIZE_FACTOR_ERROR: + return "NPP_RESIZE_FACTOR_ERROR"; + + case NPP_HAAR_CLASSIFIER_PIXEL_MATCH_ERROR: + return "NPP_HAAR_CLASSIFIER_PIXEL_MATCH_ERROR"; + +#if ((NPP_VERSION_MAJOR << 12) + (NPP_VERSION_MINOR << 4)) <= 0x5000 + + case NPP_MEMFREE_ERR: + return "NPP_MEMFREE_ERR"; + + case NPP_MEMSET_ERR: + return "NPP_MEMSET_ERR"; + + case NPP_MEMCPY_ERR: + return "NPP_MEMCPY_ERROR"; + + case NPP_MIRROR_FLIP_ERR: + return "NPP_MIRROR_FLIP_ERR"; +#else + + case NPP_MEMFREE_ERROR: + return "NPP_MEMFREE_ERROR"; + + case NPP_MEMSET_ERROR: + return "NPP_MEMSET_ERROR"; + + case NPP_MEMCPY_ERROR: + return "NPP_MEMCPY_ERROR"; + + case NPP_MIRROR_FLIP_ERROR: + return "NPP_MIRROR_FLIP_ERROR"; +#endif + + case NPP_ALIGNMENT_ERROR: + return "NPP_ALIGNMENT_ERROR"; + + case NPP_STEP_ERROR: + return "NPP_STEP_ERROR"; + + case NPP_SIZE_ERROR: + return "NPP_SIZE_ERROR"; + + case NPP_NULL_POINTER_ERROR: + return "NPP_NULL_POINTER_ERROR"; + + case NPP_CUDA_KERNEL_EXECUTION_ERROR: + return "NPP_CUDA_KERNEL_EXECUTION_ERROR"; + + case NPP_NOT_IMPLEMENTED_ERROR: + return "NPP_NOT_IMPLEMENTED_ERROR"; + + case NPP_ERROR: + return "NPP_ERROR"; + + case NPP_SUCCESS: + return "NPP_SUCCESS"; + + case NPP_WRONG_INTERSECTION_QUAD_WARNING: + return "NPP_WRONG_INTERSECTION_QUAD_WARNING"; + + case NPP_MISALIGNED_DST_ROI_WARNING: + return "NPP_MISALIGNED_DST_ROI_WARNING"; + + case NPP_AFFINE_QUAD_INCORRECT_WARNING: + return "NPP_AFFINE_QUAD_INCORRECT_WARNING"; + + case NPP_DOUBLE_SIZE_WARNING: + return "NPP_DOUBLE_SIZE_WARNING"; + + case NPP_WRONG_INTERSECTION_ROI_WARNING: + return "NPP_WRONG_INTERSECTION_ROI_WARNING"; + +#if ((NPP_VERSION_MAJOR << 12) + (NPP_VERSION_MINOR << 4)) >= 0x6000 + /* These are 6.0 or higher */ + case NPP_LUT_PALETTE_BITSIZE_ERROR: + return "NPP_LUT_PALETTE_BITSIZE_ERROR"; + + case NPP_ZC_MODE_NOT_SUPPORTED_ERROR: + return "NPP_ZC_MODE_NOT_SUPPORTED_ERROR"; + + case NPP_QUALITY_INDEX_ERROR: + return "NPP_QUALITY_INDEX_ERROR"; + + case NPP_CHANNEL_ORDER_ERROR: + return "NPP_CHANNEL_ORDER_ERROR"; + + case NPP_ZERO_MASK_VALUE_ERROR: + return "NPP_ZERO_MASK_VALUE_ERROR"; + + case NPP_NUMBER_OF_CHANNELS_ERROR: + return "NPP_NUMBER_OF_CHANNELS_ERROR"; + + case NPP_COI_ERROR: + return "NPP_COI_ERROR"; + + case NPP_DIVISOR_ERROR: + return "NPP_DIVISOR_ERROR"; + + case NPP_CHANNEL_ERROR: + return "NPP_CHANNEL_ERROR"; + + case NPP_STRIDE_ERROR: + return "NPP_STRIDE_ERROR"; + + case NPP_ANCHOR_ERROR: + return "NPP_ANCHOR_ERROR"; + + case NPP_MASK_SIZE_ERROR: + return "NPP_MASK_SIZE_ERROR"; + + case NPP_MOMENT_00_ZERO_ERROR: + return "NPP_MOMENT_00_ZERO_ERROR"; + + case NPP_THRESHOLD_NEGATIVE_LEVEL_ERROR: + return "NPP_THRESHOLD_NEGATIVE_LEVEL_ERROR"; + + case NPP_THRESHOLD_ERROR: + return "NPP_THRESHOLD_ERROR"; + + case NPP_CONTEXT_MATCH_ERROR: + return "NPP_CONTEXT_MATCH_ERROR"; + + case NPP_FFT_FLAG_ERROR: + return "NPP_FFT_FLAG_ERROR"; + + case NPP_FFT_ORDER_ERROR: + return "NPP_FFT_ORDER_ERROR"; + + case NPP_SCALE_RANGE_ERROR: + return "NPP_SCALE_RANGE_ERROR"; + + case NPP_DATA_TYPE_ERROR: + return "NPP_DATA_TYPE_ERROR"; + + case NPP_OUT_OFF_RANGE_ERROR: + return "NPP_OUT_OFF_RANGE_ERROR"; + + case NPP_DIVIDE_BY_ZERO_ERROR: + return "NPP_DIVIDE_BY_ZERO_ERROR"; + + case NPP_RANGE_ERROR: + return "NPP_RANGE_ERROR"; + + case NPP_NO_MEMORY_ERROR: + return "NPP_NO_MEMORY_ERROR"; + + case NPP_ERROR_RESERVED: + return "NPP_ERROR_RESERVED"; + + case NPP_NO_OPERATION_WARNING: + return "NPP_NO_OPERATION_WARNING"; + + case NPP_DIVIDE_BY_ZERO_WARNING: + return "NPP_DIVIDE_BY_ZERO_WARNING"; +#endif + +#if ((NPP_VERSION_MAJOR << 12) + (NPP_VERSION_MINOR << 4)) >= 0x7000 + /* These are 7.0 or higher */ + case NPP_OVERFLOW_ERROR: + return "NPP_OVERFLOW_ERROR"; + + case NPP_CORRUPTED_DATA_ERROR: + return "NPP_CORRUPTED_DATA_ERROR"; +#endif + } + + return ""; +} +#endif + +template +void check(T result, char const *const func, const char *const file, + int const line) { + if (result == cudaErrorCallRequiresNewerDriver) { + return; + } + if (result) { + fprintf(stderr, "CUDA error at %s:%d code=%d(%s) \"%s\" \n", file, line, + static_cast(result), _cudaGetErrorEnum(result), func); + exit(EXIT_FAILURE); + } +} + +#ifdef __DRIVER_TYPES_H__ +// This will output the proper CUDA error strings in the event +// that a CUDA host call returns an error +#define checkCudaErrors(val) check((val), #val, __FILE__, __LINE__) + +// This will output the proper error string when calling cudaGetLastError +#define getLastCudaError(msg) __getLastCudaError(msg, __FILE__, __LINE__) + +inline void __getLastCudaError(const char *errorMessage, const char *file, + const int line) { + cudaError_t err = cudaGetLastError(); + + if (cudaSuccess != err) { + fprintf(stderr, + "%s(%i) : getLastCudaError() CUDA error :" + " %s : (%d) %s.\n", + file, line, errorMessage, static_cast(err), + cudaGetErrorString(err)); + exit(EXIT_FAILURE); + } +} + +// This will only print the proper error string when calling cudaGetLastError +// but not exit program incase error detected. +#define printLastCudaError(msg) __printLastCudaError(msg, __FILE__, __LINE__) + +inline void __printLastCudaError(const char *errorMessage, const char *file, + const int line) { + cudaError_t err = cudaGetLastError(); + + if (cudaSuccess != err) { + fprintf(stderr, + "%s(%i) : getLastCudaError() CUDA error :" + " %s : (%d) %s.\n", + file, line, errorMessage, static_cast(err), + cudaGetErrorString(err)); + } +} +#endif + +#ifndef MAX +#define MAX(a, b) (a > b ? a : b) +#endif + +// Float To Int conversion +inline int ftoi(float value) { + return (value >= 0 ? static_cast(value + 0.5) + : static_cast(value - 0.5)); +} + +// Beginning of GPU Architecture definitions +inline int _ConvertSMVer2Cores(int major, int minor) { + // Defines for GPU Architecture types (using the SM version to determine + // the # of cores per SM + typedef struct { + int SM; // 0xMm (hexidecimal notation), M = SM Major version, + // and m = SM minor version + int Cores; + } sSMtoCores; + + sSMtoCores nGpuArchCoresPerSM[] = { + {0x30, 192}, + {0x32, 192}, + {0x35, 192}, + {0x37, 192}, + {0x50, 128}, + {0x52, 128}, + {0x53, 128}, + {0x60, 64}, + {0x61, 128}, + {0x62, 128}, + {0x70, 64}, + {0x72, 64}, + {0x75, 64}, + {0x80, 64}, + {0x86, 128}, + {0x87, 128}, + {0x89, 128}, + {0x90, 128}, + {-1, -1}}; + + int index = 0; + + while (nGpuArchCoresPerSM[index].SM != -1) { + if (nGpuArchCoresPerSM[index].SM == ((major << 4) + minor)) { + return nGpuArchCoresPerSM[index].Cores; + } + + index++; + } + + // If we don't find the values, we default use the previous one + // to run properly + printf( + "MapSMtoCores for SM %d.%d is undefined." + " Default to use %d Cores/SM\n", + major, minor, nGpuArchCoresPerSM[index - 1].Cores); + return nGpuArchCoresPerSM[index - 1].Cores; +} + +inline const char* _ConvertSMVer2ArchName(int major, int minor) { + // Defines for GPU Architecture types (using the SM version to determine + // the GPU Arch name) + typedef struct { + int SM; // 0xMm (hexidecimal notation), M = SM Major version, + // and m = SM minor version + const char* name; + } sSMtoArchName; + + sSMtoArchName nGpuArchNameSM[] = { + {0x30, "Kepler"}, + {0x32, "Kepler"}, + {0x35, "Kepler"}, + {0x37, "Kepler"}, + {0x50, "Maxwell"}, + {0x52, "Maxwell"}, + {0x53, "Maxwell"}, + {0x60, "Pascal"}, + {0x61, "Pascal"}, + {0x62, "Pascal"}, + {0x70, "Volta"}, + {0x72, "Xavier"}, + {0x75, "Turing"}, + {0x80, "Ampere"}, + {0x86, "Ampere"}, + {0x87, "Ampere"}, + {0x89, "Ada"}, + {0x90, "Hopper"}, + {-1, "Graphics Device"}}; + + int index = 0; + + while (nGpuArchNameSM[index].SM != -1) { + if (nGpuArchNameSM[index].SM == ((major << 4) + minor)) { + return nGpuArchNameSM[index].name; + } + + index++; + } + + // If we don't find the values, we default use the previous one + // to run properly + printf( + "MapSMtoArchName for SM %d.%d is undefined." + " Default to use %s\n", + major, minor, nGpuArchNameSM[index - 1].name); + return nGpuArchNameSM[index - 1].name; +} + // end of GPU Architecture definitions + +#ifdef __CUDA_RUNTIME_H__ +// General GPU Device CUDA Initialization +inline int gpuDeviceInit(int devID) { + int device_count; + checkCudaErrors(cudaGetDeviceCount(&device_count)); + + if (device_count == 0) { + fprintf(stderr, + "gpuDeviceInit() CUDA error: " + "no devices supporting CUDA.\n"); + exit(EXIT_FAILURE); + } + + if (devID < 0) { + devID = 0; + } + + if (devID > device_count - 1) { + fprintf(stderr, "\n"); + fprintf(stderr, ">> %d CUDA capable GPU device(s) detected. <<\n", + device_count); + fprintf(stderr, + ">> gpuDeviceInit (-device=%d) is not a valid" + " GPU device. <<\n", + devID); + fprintf(stderr, "\n"); + return -devID; + } + + int computeMode = -1, major = 0, minor = 0; + checkCudaErrors(cudaDeviceGetAttribute(&computeMode, cudaDevAttrComputeMode, devID)); + checkCudaErrors(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, devID)); + checkCudaErrors(cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, devID)); + if (computeMode == cudaComputeModeProhibited) { + fprintf(stderr, + "Error: device is running in , no threads can use cudaSetDevice().\n"); + return -1; + } + + if (major < 1) { + fprintf(stderr, "gpuDeviceInit(): GPU device does not support CUDA.\n"); + exit(EXIT_FAILURE); + } + + checkCudaErrors(cudaSetDevice(devID)); + printf("gpuDeviceInit() CUDA Device [%d]: \"%s\n", devID, _ConvertSMVer2ArchName(major, minor)); + + return devID; +} + +// This function returns the best GPU (with maximum GFLOPS) +inline int gpuGetMaxGflopsDeviceId() { + int current_device = 0, sm_per_multiproc = 0; + int max_perf_device = 0; + int device_count = 0; + int devices_prohibited = 0; + + uint64_t max_compute_perf = 0; + checkCudaErrors(cudaGetDeviceCount(&device_count)); + + if (device_count == 0) { + fprintf(stderr, + "gpuGetMaxGflopsDeviceId() CUDA error:" + " no devices supporting CUDA.\n"); + exit(EXIT_FAILURE); + } + + // Find the best CUDA capable GPU device + current_device = 0; + + while (current_device < device_count) { + int computeMode = -1, major = 0, minor = 0; + checkCudaErrors(cudaDeviceGetAttribute(&computeMode, cudaDevAttrComputeMode, current_device)); + checkCudaErrors(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, current_device)); + checkCudaErrors(cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, current_device)); + + // If this GPU is not running on Compute Mode prohibited, + // then we can add it to the list + if (computeMode != cudaComputeModeProhibited) { + if (major == 9999 && minor == 9999) { + sm_per_multiproc = 1; + } else { + sm_per_multiproc = + _ConvertSMVer2Cores(major, minor); + } + int multiProcessorCount = 0, clockRate = 0; + checkCudaErrors(cudaDeviceGetAttribute(&multiProcessorCount, cudaDevAttrMultiProcessorCount, current_device)); + cudaError_t result = cudaDeviceGetAttribute(&clockRate, cudaDevAttrClockRate, current_device); + if (result != cudaSuccess) { + // If cudaDevAttrClockRate attribute is not supported we + // set clockRate as 1, to consider GPU with most SMs and CUDA Cores. + if(result == cudaErrorInvalidValue) { + clockRate = 1; + } + else { + fprintf(stderr, "CUDA error at %s:%d code=%d(%s) \n", __FILE__, __LINE__, + static_cast(result), _cudaGetErrorEnum(result)); + exit(EXIT_FAILURE); + } + } + uint64_t compute_perf = (uint64_t)multiProcessorCount * sm_per_multiproc * clockRate; + + if (compute_perf > max_compute_perf) { + max_compute_perf = compute_perf; + max_perf_device = current_device; + } + } else { + devices_prohibited++; + } + + ++current_device; + } + + if (devices_prohibited == device_count) { + fprintf(stderr, + "gpuGetMaxGflopsDeviceId() CUDA error:" + " all devices have compute mode prohibited.\n"); + exit(EXIT_FAILURE); + } + + return max_perf_device; +} + +// Initialization code to find the best CUDA Device +inline int findCudaDevice(int argc, const char **argv) { + int devID = 0; + + // If the command-line has a device number specified, use it + if (checkCmdLineFlag(argc, argv, "device")) { + devID = getCmdLineArgumentInt(argc, argv, "device="); + + if (devID < 0) { + printf("Invalid command line parameter\n "); + exit(EXIT_FAILURE); + } else { + devID = gpuDeviceInit(devID); + + if (devID < 0) { + printf("exiting...\n"); + exit(EXIT_FAILURE); + } + } + } else { + // Otherwise pick the device with highest Gflops/s + devID = gpuGetMaxGflopsDeviceId(); + checkCudaErrors(cudaSetDevice(devID)); + int major = 0, minor = 0; + checkCudaErrors(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, devID)); + checkCudaErrors(cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, devID)); + printf("GPU Device %d: \"%s\" with compute capability %d.%d\n\n", + devID, _ConvertSMVer2ArchName(major, minor), major, minor); + + } + + return devID; +} + +inline int findIntegratedGPU() { + int current_device = 0; + int device_count = 0; + int devices_prohibited = 0; + + checkCudaErrors(cudaGetDeviceCount(&device_count)); + + if (device_count == 0) { + fprintf(stderr, "CUDA error: no devices supporting CUDA.\n"); + exit(EXIT_FAILURE); + } + + // Find the integrated GPU which is compute capable + while (current_device < device_count) { + int computeMode = -1, integrated = -1; + checkCudaErrors(cudaDeviceGetAttribute(&computeMode, cudaDevAttrComputeMode, current_device)); + checkCudaErrors(cudaDeviceGetAttribute(&integrated, cudaDevAttrIntegrated, current_device)); + // If GPU is integrated and is not running on Compute Mode prohibited, + // then cuda can map to GLES resource + if (integrated && (computeMode != cudaComputeModeProhibited)) { + checkCudaErrors(cudaSetDevice(current_device)); + + int major = 0, minor = 0; + checkCudaErrors(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, current_device)); + checkCudaErrors(cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, current_device)); + printf("GPU Device %d: \"%s\" with compute capability %d.%d\n\n", + current_device, _ConvertSMVer2ArchName(major, minor), major, minor); + + return current_device; + } else { + devices_prohibited++; + } + + current_device++; + } + + if (devices_prohibited == device_count) { + fprintf(stderr, + "CUDA error:" + " No GLES-CUDA Interop capable GPU found.\n"); + exit(EXIT_FAILURE); + } + + return -1; +} + +// General check for CUDA GPU SM Capabilities +inline bool checkCudaCapabilities(int major_version, int minor_version) { + int dev; + int major = 0, minor = 0; + + checkCudaErrors(cudaGetDevice(&dev)); + checkCudaErrors(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev)); + checkCudaErrors(cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev)); + + if ((major > major_version) || + (major == major_version && + minor >= minor_version)) { + printf(" Device %d: <%16s >, Compute SM %d.%d detected\n", dev, + _ConvertSMVer2ArchName(major, minor), major, minor); + return true; + } else { + printf( + " No GPU device was found that can support " + "CUDA compute capability %d.%d.\n", + major_version, minor_version); + return false; + } +} +#endif + + // end of CUDA Helper Functions + +#endif // COMMON_HELPER_CUDA_H_ diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_functions.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_functions.h new file mode 100644 index 0000000..2975ddb --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_functions.h @@ -0,0 +1,59 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// These are helper functions for the SDK samples (string parsing, +// timers, image helpers, etc) +#ifndef COMMON_HELPER_FUNCTIONS_H_ +#define COMMON_HELPER_FUNCTIONS_H_ + +#ifdef WIN32 +#pragma warning(disable : 4996) +#endif + +// includes, project +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +// includes, timer, string parsing, image helpers +#include // helper functions for image compare, dump, data comparisons +#include // helper functions for string parsing +#include // helper functions for timers + +#ifndef EXIT_WAIVED +#define EXIT_WAIVED 2 +#endif + +#endif // COMMON_HELPER_FUNCTIONS_H_ diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_image.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_image.h new file mode 100644 index 0000000..9b7edc0 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_image.h @@ -0,0 +1,1001 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// These are helper functions for the SDK samples (image,bitmap) +#ifndef COMMON_HELPER_IMAGE_H_ +#define COMMON_HELPER_IMAGE_H_ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifndef MIN +#define MIN(a, b) ((a < b) ? a : b) +#endif +#ifndef MAX +#define MAX(a, b) ((a > b) ? a : b) +#endif + +#ifndef EXIT_WAIVED +#define EXIT_WAIVED 2 +#endif + +#include + +// namespace unnamed (internal) +namespace helper_image_internal { +//! size of PGM file header +const unsigned int PGMHeaderSize = 0x40; + +// types + +//! Data converter from unsigned char / unsigned byte to type T +template +struct ConverterFromUByte; + +//! Data converter from unsigned char / unsigned byte +template <> +struct ConverterFromUByte { + //! Conversion operator + //! @return converted value + //! @param val value to convert + float operator()(const unsigned char &val) { + return static_cast(val); + } +}; + +//! Data converter from unsigned char / unsigned byte to float +template <> +struct ConverterFromUByte { + //! Conversion operator + //! @return converted value + //! @param val value to convert + float operator()(const unsigned char &val) { + return static_cast(val) / 255.0f; + } +}; + +//! Data converter from unsigned char / unsigned byte to type T +template +struct ConverterToUByte; + +//! Data converter from unsigned char / unsigned byte to unsigned int +template <> +struct ConverterToUByte { + //! Conversion operator (essentially a passthru + //! @return converted value + //! @param val value to convert + unsigned char operator()(const unsigned char &val) { return val; } +}; + +//! Data converter from unsigned char / unsigned byte to unsigned int +template <> +struct ConverterToUByte { + //! Conversion operator + //! @return converted value + //! @param val value to convert + unsigned char operator()(const float &val) { + return static_cast(val * 255.0f); + } +}; +} // namespace helper_image_internal + +#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) +#ifndef FOPEN +#define FOPEN(fHandle, filename, mode) fopen_s(&fHandle, filename, mode) +#endif +#ifndef FOPEN_FAIL +#define FOPEN_FAIL(result) (result != 0) +#endif +#ifndef SSCANF +#define SSCANF sscanf_s +#endif +#else +#ifndef FOPEN +#define FOPEN(fHandle, filename, mode) (fHandle = fopen(filename, mode)) +#endif +#ifndef FOPEN_FAIL +#define FOPEN_FAIL(result) (result == NULL) +#endif +#ifndef SSCANF +#define SSCANF sscanf +#endif +#endif + +inline bool __loadPPM(const char *file, unsigned char **data, unsigned int *w, + unsigned int *h, unsigned int *channels) { + FILE *fp = NULL; + + if (FOPEN_FAIL(FOPEN(fp, file, "rb"))) { + std::cerr << "__LoadPPM() : Failed to open file: " << file << std::endl; + return false; + } + + // check header + char header[helper_image_internal::PGMHeaderSize]; + + if (fgets(header, helper_image_internal::PGMHeaderSize, fp) == NULL) { + std::cerr << "__LoadPPM() : reading PGM header returned NULL" << std::endl; + return false; + } + + if (strncmp(header, "P5", 2) == 0) { + *channels = 1; + } else if (strncmp(header, "P6", 2) == 0) { + *channels = 3; + } else { + std::cerr << "__LoadPPM() : File is not a PPM or PGM image" << std::endl; + *channels = 0; + return false; + } + + // parse header, read maxval, width and height + unsigned int width = 0; + unsigned int height = 0; + unsigned int maxval = 0; + unsigned int i = 0; + + while (i < 3) { + if (fgets(header, helper_image_internal::PGMHeaderSize, fp) == NULL) { + std::cerr << "__LoadPPM() : reading PGM header returned NULL" + << std::endl; + return false; + } + + if (header[0] == '#') { + continue; + } + + if (i == 0) { + i += SSCANF(header, "%u %u %u", &width, &height, &maxval); + } else if (i == 1) { + i += SSCANF(header, "%u %u", &height, &maxval); + } else if (i == 2) { + i += SSCANF(header, "%u", &maxval); + } + } + + // check if given handle for the data is initialized + if (NULL != *data) { + if (*w != width || *h != height) { + std::cerr << "__LoadPPM() : Invalid image dimensions." << std::endl; + } + } else { + *data = (unsigned char *)malloc(sizeof(unsigned char) * width * height * + *channels); + *w = width; + *h = height; + } + + // read and close file + if (fread(*data, sizeof(unsigned char), width * height * *channels, fp) == + 0) { + std::cerr << "__LoadPPM() read data returned error." << std::endl; + } + + fclose(fp); + + return true; +} + +template +inline bool sdkLoadPGM(const char *file, T **data, unsigned int *w, + unsigned int *h) { + unsigned char *idata = NULL; + unsigned int channels; + + if (true != __loadPPM(file, &idata, w, h, &channels)) { + return false; + } + + unsigned int size = *w * *h * channels; + + // initialize mem if necessary + // the correct size is checked / set in loadPGMc() + if (NULL == *data) { + *data = reinterpret_cast(malloc(sizeof(T) * size)); + } + + // copy and cast data + std::transform(idata, idata + size, *data, + helper_image_internal::ConverterFromUByte()); + + free(idata); + + return true; +} + +template +inline bool sdkLoadPPM4(const char *file, T **data, unsigned int *w, + unsigned int *h) { + unsigned char *idata = 0; + unsigned int channels; + + if (__loadPPM(file, &idata, w, h, &channels)) { + // pad 4th component + int size = *w * *h; + // keep the original pointer + unsigned char *idata_orig = idata; + *data = reinterpret_cast(malloc(sizeof(T) * size * 4)); + unsigned char *ptr = *data; + + for (int i = 0; i < size; i++) { + *ptr++ = *idata++; + *ptr++ = *idata++; + *ptr++ = *idata++; + *ptr++ = 0; + } + + free(idata_orig); + return true; + } else { + free(idata); + return false; + } +} + +inline bool __savePPM(const char *file, unsigned char *data, unsigned int w, + unsigned int h, unsigned int channels) { + assert(NULL != data); + assert(w > 0); + assert(h > 0); + + std::fstream fh(file, std::fstream::out | std::fstream::binary); + + if (fh.bad()) { + std::cerr << "__savePPM() : Opening file failed." << std::endl; + return false; + } + + if (channels == 1) { + fh << "P5\n"; + } else if (channels == 3) { + fh << "P6\n"; + } else { + std::cerr << "__savePPM() : Invalid number of channels." << std::endl; + return false; + } + + fh << w << "\n" << h << "\n" << 0xff << std::endl; + + for (unsigned int i = 0; (i < (w * h * channels)) && fh.good(); ++i) { + fh << data[i]; + } + + fh.flush(); + + if (fh.bad()) { + std::cerr << "__savePPM() : Writing data failed." << std::endl; + return false; + } + + fh.close(); + + return true; +} + +template +inline bool sdkSavePGM(const char *file, T *data, unsigned int w, + unsigned int h) { + unsigned int size = w * h; + unsigned char *idata = (unsigned char *)malloc(sizeof(unsigned char) * size); + + std::transform(data, data + size, idata, + helper_image_internal::ConverterToUByte()); + + // write file + bool result = __savePPM(file, idata, w, h, 1); + + // cleanup + free(idata); + + return result; +} + +inline bool sdkSavePPM4ub(const char *file, unsigned char *data, unsigned int w, + unsigned int h) { + // strip 4th component + int size = w * h; + unsigned char *ndata = + (unsigned char *)malloc(sizeof(unsigned char) * size * 3); + unsigned char *ptr = ndata; + + for (int i = 0; i < size; i++) { + *ptr++ = *data++; + *ptr++ = *data++; + *ptr++ = *data++; + data++; + } + + bool result = __savePPM(file, ndata, w, h, 3); + free(ndata); + return result; +} + +////////////////////////////////////////////////////////////////////////////// +//! Read file \filename and return the data +//! @return bool if reading the file succeeded, otherwise false +//! @param filename name of the source file +//! @param data uninitialized pointer, returned initialized and pointing to +//! the data read +//! @param len number of data elements in data, -1 on error +////////////////////////////////////////////////////////////////////////////// +template +inline bool sdkReadFile(const char *filename, T **data, unsigned int *len, + bool verbose) { + // check input arguments + assert(NULL != filename); + assert(NULL != len); + + // intermediate storage for the data read + std::vector data_read; + + // open file for reading + FILE *fh = NULL; + + // check if filestream is valid + if (FOPEN_FAIL(FOPEN(fh, filename, "r"))) { + printf("Unable to open input file: %s\n", filename); + return false; + } + + // read all data elements + T token; + + while (!feof(fh)) { + fscanf(fh, "%f", &token); + data_read.push_back(token); + } + + // the last element is read twice + data_read.pop_back(); + fclose(fh); + + // check if the given handle is already initialized + if (NULL != *data) { + if (*len != data_read.size()) { + std::cerr << "sdkReadFile() : Initialized memory given but " + << "size mismatch with signal read " + << "(data read / data init = " << (unsigned int)data_read.size() + << " / " << *len << ")" << std::endl; + + return false; + } + } else { + // allocate storage for the data read + *data = reinterpret_cast(malloc(sizeof(T) * data_read.size())); + // store signal size + *len = static_cast(data_read.size()); + } + + // copy data + memcpy(*data, &data_read.front(), sizeof(T) * data_read.size()); + + return true; +} + +////////////////////////////////////////////////////////////////////////////// +//! Read file \filename and return the data +//! @return bool if reading the file succeeded, otherwise false +//! @param filename name of the source file +//! @param data uninitialized pointer, returned initialized and pointing to +//! the data read +//! @param len number of data elements in data, -1 on error +////////////////////////////////////////////////////////////////////////////// +template +inline bool sdkReadFileBlocks(const char *filename, T **data, unsigned int *len, + unsigned int block_num, unsigned int block_size, + bool verbose) { + // check input arguments + assert(NULL != filename); + assert(NULL != len); + + // open file for reading + FILE *fh = fopen(filename, "rb"); + + if (fh == NULL && verbose) { + std::cerr << "sdkReadFile() : Opening file failed." << std::endl; + return false; + } + + // check if the given handle is already initialized + // allocate storage for the data read + data[block_num] = reinterpret_cast(malloc(block_size)); + + // read all data elements + fseek(fh, block_num * block_size, SEEK_SET); + *len = fread(data[block_num], sizeof(T), block_size / sizeof(T), fh); + + fclose(fh); + + return true; +} + +////////////////////////////////////////////////////////////////////////////// +//! Write a data file \filename +//! @return true if writing the file succeeded, otherwise false +//! @param filename name of the source file +//! @param data data to write +//! @param len number of data elements in data, -1 on error +//! @param epsilon epsilon for comparison +////////////////////////////////////////////////////////////////////////////// +template +inline bool sdkWriteFile(const char *filename, const T *data, unsigned int len, + const S epsilon, bool verbose, bool append = false) { + assert(NULL != filename); + assert(NULL != data); + + // open file for writing + // if (append) { + std::fstream fh(filename, std::fstream::out | std::fstream::ate); + + if (verbose) { + std::cerr << "sdkWriteFile() : Open file " << filename + << " for write/append." << std::endl; + } + + /* } else { + std::fstream fh(filename, std::fstream::out); + if (verbose) { + std::cerr << "sdkWriteFile() : Open file " << filename << " for + write." << std::endl; + } + } + */ + + // check if filestream is valid + if (!fh.good()) { + if (verbose) { + std::cerr << "sdkWriteFile() : Opening file failed." << std::endl; + } + + return false; + } + + // first write epsilon + fh << "# " << epsilon << "\n"; + + // write data + for (unsigned int i = 0; (i < len) && (fh.good()); ++i) { + fh << data[i] << ' '; + } + + // Check if writing succeeded + if (!fh.good()) { + if (verbose) { + std::cerr << "sdkWriteFile() : Writing file failed." << std::endl; + } + + return false; + } + + // file ends with nl + fh << std::endl; + + return true; +} + +////////////////////////////////////////////////////////////////////////////// +//! Compare two arrays of arbitrary type +//! @return true if \a reference and \a data are identical, otherwise false +//! @param reference timer_interface to the reference data / gold image +//! @param data handle to the computed data +//! @param len number of elements in reference and data +//! @param epsilon epsilon to use for the comparison +////////////////////////////////////////////////////////////////////////////// +template +inline bool compareData(const T *reference, const T *data, + const unsigned int len, const S epsilon, + const float threshold) { + assert(epsilon >= 0); + + bool result = true; + unsigned int error_count = 0; + + for (unsigned int i = 0; i < len; ++i) { + float diff = static_cast(reference[i]) - static_cast(data[i]); + bool comp = (diff <= epsilon) && (diff >= -epsilon); + result &= comp; + + error_count += !comp; + +#if 0 + + if (!comp) { + std::cerr << "ERROR, i = " << i << ",\t " + << reference[i] << " / " + << data[i] + << " (reference / data)\n"; + } + +#endif + } + + if (threshold == 0.0f) { + return (result) ? true : false; + } else { + if (error_count) { + printf("%4.2f(%%) of bytes mismatched (count=%d)\n", + static_cast(error_count) * 100 / static_cast(len), + error_count); + } + + return (len * threshold > error_count) ? true : false; + } +} + +#ifndef __MIN_EPSILON_ERROR +#define __MIN_EPSILON_ERROR 1e-3f +#endif + +////////////////////////////////////////////////////////////////////////////// +//! Compare two arrays of arbitrary type +//! @return true if \a reference and \a data are identical, otherwise false +//! @param reference handle to the reference data / gold image +//! @param data handle to the computed data +//! @param len number of elements in reference and data +//! @param epsilon epsilon to use for the comparison +//! @param epsilon threshold % of (# of bytes) for pass/fail +////////////////////////////////////////////////////////////////////////////// +template +inline bool compareDataAsFloatThreshold(const T *reference, const T *data, + const unsigned int len, const S epsilon, + const float threshold) { + assert(epsilon >= 0); + + // If we set epsilon to be 0, let's set a minimum threshold + float max_error = MAX((float)epsilon, __MIN_EPSILON_ERROR); + int error_count = 0; + bool result = true; + + for (unsigned int i = 0; i < len; ++i) { + float diff = + fabs(static_cast(reference[i]) - static_cast(data[i])); + bool comp = (diff < max_error); + result &= comp; + + if (!comp) { + error_count++; + } + } + + if (threshold == 0.0f) { + if (error_count) { + printf("total # of errors = %d\n", error_count); + } + + return (error_count == 0) ? true : false; + } else { + if (error_count) { + printf("%4.2f(%%) of bytes mismatched (count=%d)\n", + static_cast(error_count) * 100 / static_cast(len), + error_count); + } + + return ((len * threshold > error_count) ? true : false); + } +} + +inline void sdkDumpBin(void *data, unsigned int bytes, const char *filename) { + printf("sdkDumpBin: <%s>\n", filename); + FILE *fp; + FOPEN(fp, filename, "wb"); + fwrite(data, bytes, 1, fp); + fflush(fp); + fclose(fp); +} + +inline bool sdkCompareBin2BinUint(const char *src_file, const char *ref_file, + unsigned int nelements, const float epsilon, + const float threshold, char *exec_path) { + unsigned int *src_buffer, *ref_buffer; + FILE *src_fp = NULL, *ref_fp = NULL; + + uint64_t error_count = 0; + size_t fsize = 0; + + if (FOPEN_FAIL(FOPEN(src_fp, src_file, "rb"))) { + printf("compareBin2Bin unable to open src_file: %s\n", + src_file); + error_count++; + } + + char *ref_file_path = sdkFindFilePath(ref_file, exec_path); + + if (ref_file_path == NULL) { + printf("compareBin2Bin unable to find <%s> in <%s>\n", + ref_file, exec_path); + printf(">>> Check info.xml and [project//data] folder <%s> <<<\n", + ref_file); + printf("Aborting comparison!\n"); + printf(" FAILED\n"); + error_count++; + + if (src_fp) { + fclose(src_fp); + } + + if (ref_fp) { + fclose(ref_fp); + } + } else { + if (FOPEN_FAIL(FOPEN(ref_fp, ref_file_path, "rb"))) { + printf( + "compareBin2Bin " + " unable to open ref_file: %s\n", + ref_file_path); + error_count++; + } + + if (src_fp && ref_fp) { + src_buffer = (unsigned int *)malloc(nelements * sizeof(unsigned int)); + ref_buffer = (unsigned int *)malloc(nelements * sizeof(unsigned int)); + + fsize = fread(src_buffer, nelements, sizeof(unsigned int), src_fp); + fsize = fread(ref_buffer, nelements, sizeof(unsigned int), ref_fp); + + printf( + "> compareBin2Bin nelements=%d," + " epsilon=%4.2f, threshold=%4.2f\n", + nelements, epsilon, threshold); + printf(" src_file <%s>, size=%d bytes\n", src_file, + static_cast(fsize)); + printf(" ref_file <%s>, size=%d bytes\n", ref_file_path, + static_cast(fsize)); + + if (!compareData(ref_buffer, src_buffer, nelements, + epsilon, threshold)) { + error_count++; + } + + fclose(src_fp); + fclose(ref_fp); + + free(src_buffer); + free(ref_buffer); + } else { + if (src_fp) { + fclose(src_fp); + } + + if (ref_fp) { + fclose(ref_fp); + } + } + } + + if (error_count == 0) { + printf(" OK\n"); + } else { + printf(" FAILURE: %d errors...\n", (unsigned int)error_count); + } + + return (error_count == 0); // returns true if all pixels pass +} + +inline bool sdkCompareBin2BinFloat(const char *src_file, const char *ref_file, + unsigned int nelements, const float epsilon, + const float threshold, char *exec_path) { + float *src_buffer = NULL, *ref_buffer = NULL; + FILE *src_fp = NULL, *ref_fp = NULL; + size_t fsize = 0; + + uint64_t error_count = 0; + + if (FOPEN_FAIL(FOPEN(src_fp, src_file, "rb"))) { + printf("compareBin2Bin unable to open src_file: %s\n", src_file); + error_count = 1; + } + + char *ref_file_path = sdkFindFilePath(ref_file, exec_path); + + if (ref_file_path == NULL) { + printf("compareBin2Bin unable to find <%s> in <%s>\n", ref_file, + exec_path); + printf(">>> Check info.xml and [project//data] folder <%s> <<<\n", + exec_path); + printf("Aborting comparison!\n"); + printf(" FAILED\n"); + error_count++; + + if (src_fp) { + fclose(src_fp); + } + + if (ref_fp) { + fclose(ref_fp); + } + } else { + if (FOPEN_FAIL(FOPEN(ref_fp, ref_file_path, "rb"))) { + printf("compareBin2Bin unable to open ref_file: %s\n", + ref_file_path); + error_count = 1; + } + + if (src_fp && ref_fp) { + src_buffer = reinterpret_cast(malloc(nelements * sizeof(float))); + ref_buffer = reinterpret_cast(malloc(nelements * sizeof(float))); + + printf( + "> compareBin2Bin nelements=%d, epsilon=%4.2f," + " threshold=%4.2f\n", + nelements, epsilon, threshold); + fsize = fread(src_buffer, sizeof(float), nelements, src_fp); + printf(" src_file <%s>, size=%d bytes\n", src_file, + static_cast(fsize * sizeof(float))); + fsize = fread(ref_buffer, sizeof(float), nelements, ref_fp); + printf(" ref_file <%s>, size=%d bytes\n", ref_file_path, + static_cast(fsize * sizeof(float))); + + if (!compareDataAsFloatThreshold( + ref_buffer, src_buffer, nelements, epsilon, threshold)) { + error_count++; + } + + fclose(src_fp); + fclose(ref_fp); + + free(src_buffer); + free(ref_buffer); + } else { + if (src_fp) { + fclose(src_fp); + } + + if (ref_fp) { + fclose(ref_fp); + } + } + } + + if (error_count == 0) { + printf(" OK\n"); + } else { + printf(" FAILURE: %d errors...\n", (unsigned int)error_count); + } + + return (error_count == 0); // returns true if all pixels pass +} + +inline bool sdkCompareL2fe(const float *reference, const float *data, + const unsigned int len, const float epsilon) { + assert(epsilon >= 0); + + float error = 0; + float ref = 0; + + for (unsigned int i = 0; i < len; ++i) { + float diff = reference[i] - data[i]; + error += diff * diff; + ref += reference[i] * reference[i]; + } + + float normRef = sqrtf(ref); + + if (fabs(ref) < 1e-7) { +#ifdef _DEBUG + std::cerr << "ERROR, reference l2-norm is 0\n"; +#endif + return false; + } + + float normError = sqrtf(error); + error = normError / normRef; + bool result = error < epsilon; +#ifdef _DEBUG + + if (!result) { + std::cerr << "ERROR, l2-norm error " << error << " is greater than epsilon " + << epsilon << "\n"; + } + +#endif + + return result; +} + +inline bool sdkLoadPPMub(const char *file, unsigned char **data, + unsigned int *w, unsigned int *h) { + unsigned int channels; + return __loadPPM(file, data, w, h, &channels); +} + +inline bool sdkLoadPPM4ub(const char *file, unsigned char **data, + unsigned int *w, unsigned int *h) { + unsigned char *idata = 0; + unsigned int channels; + + if (__loadPPM(file, &idata, w, h, &channels)) { + // pad 4th component + int size = *w * *h; + // keep the original pointer + unsigned char *idata_orig = idata; + *data = (unsigned char *)malloc(sizeof(unsigned char) * size * 4); + unsigned char *ptr = *data; + + for (int i = 0; i < size; i++) { + *ptr++ = *idata++; + *ptr++ = *idata++; + *ptr++ = *idata++; + *ptr++ = 0; + } + + free(idata_orig); + return true; + } else { + free(idata); + return false; + } +} + +inline bool sdkComparePPM(const char *src_file, const char *ref_file, + const float epsilon, const float threshold, + bool verboseErrors) { + unsigned char *src_data, *ref_data; + uint64_t error_count = 0; + unsigned int ref_width, ref_height; + unsigned int src_width, src_height; + + if (src_file == NULL || ref_file == NULL) { + if (verboseErrors) { + std::cerr << "PPMvsPPM: src_file or ref_file is NULL." + " Aborting comparison\n"; + } + + return false; + } + + if (verboseErrors) { + std::cerr << "> Compare (a)rendered: <" << src_file << ">\n"; + std::cerr << "> (b)reference: <" << ref_file << ">\n"; + } + + if (sdkLoadPPM4ub(ref_file, &ref_data, &ref_width, &ref_height) != true) { + if (verboseErrors) { + std::cerr << "PPMvsPPM: unable to load ref image file: " << ref_file + << "\n"; + } + + return false; + } + + if (sdkLoadPPM4ub(src_file, &src_data, &src_width, &src_height) != true) { + std::cerr << "PPMvsPPM: unable to load src image file: " << src_file + << "\n"; + return false; + } + + if (src_height != ref_height || src_width != ref_width) { + if (verboseErrors) { + std::cerr << "PPMvsPPM: source and ref size mismatch (" << src_width + << "," << src_height << ")vs(" << ref_width << "," << ref_height + << ")\n"; + } + } + + if (verboseErrors) { + std::cerr << "PPMvsPPM: comparing images size (" << src_width << "," + << src_height << ") epsilon(" << epsilon << "), threshold(" + << threshold * 100 << "%)\n"; + } + + if (compareData(ref_data, src_data, src_width * src_height * 4, epsilon, + threshold) == false) { + error_count = 1; + } + + if (error_count == 0) { + if (verboseErrors) { + std::cerr << " OK\n\n"; + } + } else { + if (verboseErrors) { + std::cerr << " FAILURE! " << error_count << " errors...\n\n"; + } + } + + // returns true if all pixels pass + return (error_count == 0) ? true : false; +} + +inline bool sdkComparePGM(const char *src_file, const char *ref_file, + const float epsilon, const float threshold, + bool verboseErrors) { + unsigned char *src_data = 0, *ref_data = 0; + uint64_t error_count = 0; + unsigned int ref_width, ref_height; + unsigned int src_width, src_height; + + if (src_file == NULL || ref_file == NULL) { + if (verboseErrors) { + std::cerr << "PGMvsPGM: src_file or ref_file is NULL." + " Aborting comparison\n"; + } + + return false; + } + + if (verboseErrors) { + std::cerr << "> Compare (a)rendered: <" << src_file << ">\n"; + std::cerr << "> (b)reference: <" << ref_file << ">\n"; + } + + if (sdkLoadPPMub(ref_file, &ref_data, &ref_width, &ref_height) != true) { + if (verboseErrors) { + std::cerr << "PGMvsPGM: unable to load ref image file: " << ref_file + << "\n"; + } + + return false; + } + + if (sdkLoadPPMub(src_file, &src_data, &src_width, &src_height) != true) { + std::cerr << "PGMvsPGM: unable to load src image file: " << src_file + << "\n"; + return false; + } + + if (src_height != ref_height || src_width != ref_width) { + if (verboseErrors) { + std::cerr << "PGMvsPGM: source and ref size mismatch (" << src_width + << "," << src_height << ")vs(" << ref_width << "," << ref_height + << ")\n"; + } + } + + if (verboseErrors) + std::cerr << "PGMvsPGM: comparing images size (" << src_width << "," + << src_height << ") epsilon(" << epsilon << "), threshold(" + << threshold * 100 << "%)\n"; + + if (compareData(ref_data, src_data, src_width * src_height, epsilon, + threshold) == false) { + error_count = 1; + } + + if (error_count == 0) { + if (verboseErrors) { + std::cerr << " OK\n\n"; + } + } else { + if (verboseErrors) { + std::cerr << " FAILURE! " << error_count << " errors...\n\n"; + } + } + + // returns true if all pixels pass + return (error_count == 0) ? true : false; +} + +#endif // COMMON_HELPER_IMAGE_H_ diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_string.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_string.h new file mode 100644 index 0000000..7920060 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_string.h @@ -0,0 +1,428 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// These are helper functions for the SDK samples (string parsing, timers, etc) +#ifndef COMMON_HELPER_STRING_H_ +#define COMMON_HELPER_STRING_H_ + +#include +#include +#include +#include + +#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) +#ifndef _CRT_SECURE_NO_DEPRECATE +#define _CRT_SECURE_NO_DEPRECATE +#endif +#ifndef STRCASECMP +#define STRCASECMP _stricmp +#endif +#ifndef STRNCASECMP +#define STRNCASECMP _strnicmp +#endif +#ifndef STRCPY +#define STRCPY(sFilePath, nLength, sPath) strcpy_s(sFilePath, nLength, sPath) +#endif + +#ifndef FOPEN +#define FOPEN(fHandle, filename, mode) fopen_s(&fHandle, filename, mode) +#endif +#ifndef FOPEN_FAIL +#define FOPEN_FAIL(result) (result != 0) +#endif +#ifndef SSCANF +#define SSCANF sscanf_s +#endif +#ifndef SPRINTF +#define SPRINTF sprintf_s +#endif +#else // Linux Includes +#include +#include + +#ifndef STRCASECMP +#define STRCASECMP strcasecmp +#endif +#ifndef STRNCASECMP +#define STRNCASECMP strncasecmp +#endif +#ifndef STRCPY +#define STRCPY(sFilePath, nLength, sPath) strcpy(sFilePath, sPath) +#endif + +#ifndef FOPEN +#define FOPEN(fHandle, filename, mode) (fHandle = fopen(filename, mode)) +#endif +#ifndef FOPEN_FAIL +#define FOPEN_FAIL(result) (result == NULL) +#endif +#ifndef SSCANF +#define SSCANF sscanf +#endif +#ifndef SPRINTF +#define SPRINTF sprintf +#endif +#endif + +#ifndef EXIT_WAIVED +#define EXIT_WAIVED 2 +#endif + +// CUDA Utility Helper Functions +inline int stringRemoveDelimiter(char delimiter, const char *string) { + int string_start = 0; + + while (string[string_start] == delimiter) { + string_start++; + } + + if (string_start >= static_cast(strlen(string) - 1)) { + return 0; + } + + return string_start; +} + +inline int getFileExtension(char *filename, char **extension) { + int string_length = static_cast(strlen(filename)); + + while (filename[string_length--] != '.') { + if (string_length == 0) break; + } + + if (string_length > 0) string_length += 2; + + if (string_length == 0) + *extension = NULL; + else + *extension = &filename[string_length]; + + return string_length; +} + +inline bool checkCmdLineFlag(const int argc, const char **argv, + const char *string_ref) { + bool bFound = false; + + if (argc >= 1) { + for (int i = 1; i < argc; i++) { + int string_start = stringRemoveDelimiter('-', argv[i]); + const char *string_argv = &argv[i][string_start]; + + const char *equal_pos = strchr(string_argv, '='); + int argv_length = static_cast( + equal_pos == 0 ? strlen(string_argv) : equal_pos - string_argv); + + int length = static_cast(strlen(string_ref)); + + if (length == argv_length && + !STRNCASECMP(string_argv, string_ref, length)) { + bFound = true; + continue; + } + } + } + + return bFound; +} + +// This function wraps the CUDA Driver API into a template function +template +inline bool getCmdLineArgumentValue(const int argc, const char **argv, + const char *string_ref, T *value) { + bool bFound = false; + + if (argc >= 1) { + for (int i = 1; i < argc; i++) { + int string_start = stringRemoveDelimiter('-', argv[i]); + const char *string_argv = &argv[i][string_start]; + int length = static_cast(strlen(string_ref)); + + if (!STRNCASECMP(string_argv, string_ref, length)) { + if (length + 1 <= static_cast(strlen(string_argv))) { + int auto_inc = (string_argv[length] == '=') ? 1 : 0; + *value = (T)atoi(&string_argv[length + auto_inc]); + } + + bFound = true; + i = argc; + } + } + } + + return bFound; +} + +inline int getCmdLineArgumentInt(const int argc, const char **argv, + const char *string_ref) { + bool bFound = false; + int value = -1; + + if (argc >= 1) { + for (int i = 1; i < argc; i++) { + int string_start = stringRemoveDelimiter('-', argv[i]); + const char *string_argv = &argv[i][string_start]; + int length = static_cast(strlen(string_ref)); + + if (!STRNCASECMP(string_argv, string_ref, length)) { + if (length + 1 <= static_cast(strlen(string_argv))) { + int auto_inc = (string_argv[length] == '=') ? 1 : 0; + value = atoi(&string_argv[length + auto_inc]); + } else { + value = 0; + } + + bFound = true; + continue; + } + } + } + + if (bFound) { + return value; + } else { + return 0; + } +} + +inline float getCmdLineArgumentFloat(const int argc, const char **argv, + const char *string_ref) { + bool bFound = false; + float value = -1; + + if (argc >= 1) { + for (int i = 1; i < argc; i++) { + int string_start = stringRemoveDelimiter('-', argv[i]); + const char *string_argv = &argv[i][string_start]; + int length = static_cast(strlen(string_ref)); + + if (!STRNCASECMP(string_argv, string_ref, length)) { + if (length + 1 <= static_cast(strlen(string_argv))) { + int auto_inc = (string_argv[length] == '=') ? 1 : 0; + value = static_cast(atof(&string_argv[length + auto_inc])); + } else { + value = 0.f; + } + + bFound = true; + continue; + } + } + } + + if (bFound) { + return value; + } else { + return 0; + } +} + +inline bool getCmdLineArgumentString(const int argc, const char **argv, + const char *string_ref, + char **string_retval) { + bool bFound = false; + + if (argc >= 1) { + for (int i = 1; i < argc; i++) { + int string_start = stringRemoveDelimiter('-', argv[i]); + char *string_argv = const_cast(&argv[i][string_start]); + int length = static_cast(strlen(string_ref)); + + if (!STRNCASECMP(string_argv, string_ref, length)) { + *string_retval = &string_argv[length]; + bFound = true; + continue; + } + } + } + + if (!bFound) { + *string_retval = NULL; + } + + return bFound; +} + +////////////////////////////////////////////////////////////////////////////// +//! Find the path for a file assuming that +//! files are found in the searchPath. +//! +//! @return the path if succeeded, otherwise 0 +//! @param filename name of the file +//! @param executable_path optional absolute path of the executable +////////////////////////////////////////////////////////////////////////////// +inline char *sdkFindFilePath(const char *filename, + const char *executable_path) { + // defines a variable that is replaced with the name of the + // executable + + // Typical relative search paths to locate needed companion files (e.g. sample + // input data, or JIT source files) The origin for the relative search may be + // the .exe file, a .bat file launching an .exe, a browser .exe launching the + // .exe or .bat, etc + const char *searchPath[] = { + "./", // same dir + "./data/", // same dir + + "../../../../Samples//", // up 4 in tree + "../../../Samples//", // up 3 in tree + "../../Samples//", // up 2 in tree + + "../../../../Samples//data/", // up 4 in tree + "../../../Samples//data/", // up 3 in tree + "../../Samples//data/", // up 2 in tree + + "../../../../Samples/0_Introduction//", // up 4 in tree + "../../../Samples/0_Introduction//", // up 3 in tree + "../../Samples/0_Introduction//", // up 2 in tree + + "../../../../Samples/1_Utilities//", // up 4 in tree + "../../../Samples/1_Utilities//", // up 3 in tree + "../../Samples/1_Utilities//", // up 2 in tree + + "../../../../Samples/2_Concepts_and_Techniques//", // up 4 in tree + "../../../Samples/2_Concepts_and_Techniques//", // up 3 in tree + "../../Samples/2_Concepts_and_Techniques//", // up 2 in tree + + "../../../../Samples/3_CUDA_Features//", // up 4 in tree + "../../../Samples/3_CUDA_Features//", // up 3 in tree + "../../Samples/3_CUDA_Features//", // up 2 in tree + + "../../../../Samples/4_CUDA_Libraries//", // up 4 in tree + "../../../Samples/4_CUDA_Libraries//", // up 3 in tree + "../../Samples/4_CUDA_Libraries//", // up 2 in tree + + "../../../../Samples/5_Domain_Specific//", // up 4 in tree + "../../../Samples/5_Domain_Specific//", // up 3 in tree + "../../Samples/5_Domain_Specific//", // up 2 in tree + + "../../../../Samples/6_Performance//", // up 4 in tree + "../../../Samples/6_Performance//", // up 3 in tree + "../../Samples/6_Performance//", // up 2 in tree + + "../../../../Samples/0_Introduction//data/", // up 4 in tree + "../../../Samples/0_Introduction//data/", // up 3 in tree + "../../Samples/0_Introduction//data/", // up 2 in tree + + "../../../../Samples/1_Utilities//data/", // up 4 in tree + "../../../Samples/1_Utilities//data/", // up 3 in tree + "../../Samples/1_Utilities//data/", // up 2 in tree + + "../../../../Samples/2_Concepts_and_Techniques//data/", // up 4 in tree + "../../../Samples/2_Concepts_and_Techniques//data/", // up 3 in tree + "../../Samples/2_Concepts_and_Techniques//data/", // up 2 in tree + + "../../../../Samples/3_CUDA_Features//data/", // up 4 in tree + "../../../Samples/3_CUDA_Features//data/", // up 3 in tree + "../../Samples/3_CUDA_Features//data/", // up 2 in tree + + "../../../../Samples/4_CUDA_Libraries//data/", // up 4 in tree + "../../../Samples/4_CUDA_Libraries//data/", // up 3 in tree + "../../Samples/4_CUDA_Libraries//data/", // up 2 in tree + + "../../../../Samples/5_Domain_Specific//data/", // up 4 in tree + "../../../Samples/5_Domain_Specific//data/", // up 3 in tree + "../../Samples/5_Domain_Specific//data/", // up 2 in tree + + "../../../../Samples/6_Performance//data/", // up 4 in tree + "../../../Samples/6_Performance//data/", // up 3 in tree + "../../Samples/6_Performance//data/", // up 2 in tree + + "../../../../Common/data/", // up 4 in tree + "../../../Common/data/", // up 3 in tree + "../../Common/data/" // up 2 in tree + }; + + // Extract the executable name + std::string executable_name; + + if (executable_path != 0) { + executable_name = std::string(executable_path); + +#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) + // Windows path delimiter + size_t delimiter_pos = executable_name.find_last_of('\\'); + executable_name.erase(0, delimiter_pos + 1); + + if (executable_name.rfind(".exe") != std::string::npos) { + // we strip .exe, only if the .exe is found + executable_name.resize(executable_name.size() - 4); + } + +#else + // Linux & OSX path delimiter + size_t delimiter_pos = executable_name.find_last_of('/'); + executable_name.erase(0, delimiter_pos + 1); +#endif + } + + // Loop over all search paths and return the first hit + for (unsigned int i = 0; i < sizeof(searchPath) / sizeof(char *); ++i) { + std::string path(searchPath[i]); + size_t executable_name_pos = path.find(""); + + // If there is executable_name variable in the searchPath + // replace it with the value + if (executable_name_pos != std::string::npos) { + if (executable_path != 0) { + path.replace(executable_name_pos, strlen(""), + executable_name); + } else { + // Skip this path entry if no executable argument is given + continue; + } + } + +#ifdef _DEBUG + printf("sdkFindFilePath <%s> in %s\n", filename, path.c_str()); +#endif + + // Test if the file exists + path.append(filename); + FILE *fp; + FOPEN(fp, path.c_str(), "rb"); + + if (fp != NULL) { + fclose(fp); + // File found + // returning an allocated array here for backwards compatibility reasons + char *file_path = reinterpret_cast(malloc(path.length() + 1)); + STRCPY(file_path, path.length() + 1, path.c_str()); + return file_path; + } + + if (fp) { + fclose(fp); + } + } + + // File not found + printf("\nerror: sdkFindFilePath: file <%s> not found!\n", filename); + return 0; +} + +#endif // COMMON_HELPER_STRING_H_ diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_timer.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_timer.h new file mode 100644 index 0000000..2fe3207 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/helper_timer.h @@ -0,0 +1,465 @@ +/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// Helper Timing Functions +#ifndef COMMON_HELPER_TIMER_H_ +#define COMMON_HELPER_TIMER_H_ + +#ifndef EXIT_WAIVED +#define EXIT_WAIVED 2 +#endif + +// includes, system +#include + +// includes, project +#include + +// Definition of the StopWatch Interface, this is used if we don't want to use +// the CUT functions But rather in a self contained class interface +class StopWatchInterface { + public: + StopWatchInterface() {} + virtual ~StopWatchInterface() {} + + public: + //! Start time measurement + virtual void start() = 0; + + //! Stop time measurement + virtual void stop() = 0; + + //! Reset time counters to zero + virtual void reset() = 0; + + //! Time in msec. after start. If the stop watch is still running (i.e. there + //! was no call to stop()) then the elapsed time is returned, otherwise the + //! time between the last start() and stop call is returned + virtual float getTime() = 0; + + //! Mean time to date based on the number of times the stopwatch has been + //! _stopped_ (ie finished sessions) and the current total time + virtual float getAverageTime() = 0; +}; + +////////////////////////////////////////////////////////////////// +// Begin Stopwatch timer class definitions for all OS platforms // +////////////////////////////////////////////////////////////////// +#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) +// includes, system +#define WINDOWS_LEAN_AND_MEAN +#include +#undef min +#undef max + +//! Windows specific implementation of StopWatch +class StopWatchWin : public StopWatchInterface { + public: + //! Constructor, default + StopWatchWin() + : start_time(), + end_time(), + diff_time(0.0f), + total_time(0.0f), + running(false), + clock_sessions(0), + freq(0), + freq_set(false) { + if (!freq_set) { + // helper variable + LARGE_INTEGER temp; + + // get the tick frequency from the OS + QueryPerformanceFrequency(reinterpret_cast(&temp)); + + // convert to type in which it is needed + freq = (static_cast(temp.QuadPart)) / 1000.0; + + // rememeber query + freq_set = true; + } + } + + // Destructor + ~StopWatchWin() {} + + public: + //! Start time measurement + inline void start(); + + //! Stop time measurement + inline void stop(); + + //! Reset time counters to zero + inline void reset(); + + //! Time in msec. after start. If the stop watch is still running (i.e. there + //! was no call to stop()) then the elapsed time is returned, otherwise the + //! time between the last start() and stop call is returned + inline float getTime(); + + //! Mean time to date based on the number of times the stopwatch has been + //! _stopped_ (ie finished sessions) and the current total time + inline float getAverageTime(); + + private: + // member variables + + //! Start of measurement + LARGE_INTEGER start_time; + //! End of measurement + LARGE_INTEGER end_time; + + //! Time difference between the last start and stop + float diff_time; + + //! TOTAL time difference between starts and stops + float total_time; + + //! flag if the stop watch is running + bool running; + + //! Number of times clock has been started + //! and stopped to allow averaging + int clock_sessions; + + //! tick frequency + double freq; + + //! flag if the frequency has been set + bool freq_set; +}; + +// functions, inlined + +//////////////////////////////////////////////////////////////////////////////// +//! Start time measurement +//////////////////////////////////////////////////////////////////////////////// +inline void StopWatchWin::start() { + QueryPerformanceCounter(reinterpret_cast(&start_time)); + running = true; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Stop time measurement and increment add to the current diff_time summation +//! variable. Also increment the number of times this clock has been run. +//////////////////////////////////////////////////////////////////////////////// +inline void StopWatchWin::stop() { + QueryPerformanceCounter(reinterpret_cast(&end_time)); + diff_time = static_cast(((static_cast(end_time.QuadPart) - + static_cast(start_time.QuadPart)) / + freq)); + + total_time += diff_time; + clock_sessions++; + running = false; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Reset the timer to 0. Does not change the timer running state but does +//! recapture this point in time as the current start time if it is running. +//////////////////////////////////////////////////////////////////////////////// +inline void StopWatchWin::reset() { + diff_time = 0; + total_time = 0; + clock_sessions = 0; + + if (running) { + QueryPerformanceCounter(reinterpret_cast(&start_time)); + } +} + +//////////////////////////////////////////////////////////////////////////////// +//! Time in msec. after start. If the stop watch is still running (i.e. there +//! was no call to stop()) then the elapsed time is returned added to the +//! current diff_time sum, otherwise the current summed time difference alone +//! is returned. +//////////////////////////////////////////////////////////////////////////////// +inline float StopWatchWin::getTime() { + // Return the TOTAL time to date + float retval = total_time; + + if (running) { + LARGE_INTEGER temp; + QueryPerformanceCounter(reinterpret_cast(&temp)); + retval += static_cast(((static_cast(temp.QuadPart) - + static_cast(start_time.QuadPart)) / + freq)); + } + + return retval; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Time in msec. for a single run based on the total number of COMPLETED runs +//! and the total time. +//////////////////////////////////////////////////////////////////////////////// +inline float StopWatchWin::getAverageTime() { + return (clock_sessions > 0) ? (total_time / clock_sessions) : 0.0f; +} +#else +// Declarations for Stopwatch on Linux and Mac OSX +// includes, system +#include +#include + +//! Windows specific implementation of StopWatch +class StopWatchLinux : public StopWatchInterface { + public: + //! Constructor, default + StopWatchLinux() + : start_time(), + diff_time(0.0), + total_time(0.0), + running(false), + clock_sessions(0) {} + + // Destructor + virtual ~StopWatchLinux() {} + + public: + //! Start time measurement + inline void start(); + + //! Stop time measurement + inline void stop(); + + //! Reset time counters to zero + inline void reset(); + + //! Time in msec. after start. If the stop watch is still running (i.e. there + //! was no call to stop()) then the elapsed time is returned, otherwise the + //! time between the last start() and stop call is returned + inline float getTime(); + + //! Mean time to date based on the number of times the stopwatch has been + //! _stopped_ (ie finished sessions) and the current total time + inline float getAverageTime(); + + private: + // helper functions + + //! Get difference between start time and current time + inline float getDiffTime(); + + private: + // member variables + + //! Start of measurement + struct timeval start_time; + + //! Time difference between the last start and stop + float diff_time; + + //! TOTAL time difference between starts and stops + float total_time; + + //! flag if the stop watch is running + bool running; + + //! Number of times clock has been started + //! and stopped to allow averaging + int clock_sessions; +}; + +// functions, inlined + +//////////////////////////////////////////////////////////////////////////////// +//! Start time measurement +//////////////////////////////////////////////////////////////////////////////// +inline void StopWatchLinux::start() { + gettimeofday(&start_time, 0); + running = true; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Stop time measurement and increment add to the current diff_time summation +//! variable. Also increment the number of times this clock has been run. +//////////////////////////////////////////////////////////////////////////////// +inline void StopWatchLinux::stop() { + diff_time = getDiffTime(); + total_time += diff_time; + running = false; + clock_sessions++; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Reset the timer to 0. Does not change the timer running state but does +//! recapture this point in time as the current start time if it is running. +//////////////////////////////////////////////////////////////////////////////// +inline void StopWatchLinux::reset() { + diff_time = 0; + total_time = 0; + clock_sessions = 0; + + if (running) { + gettimeofday(&start_time, 0); + } +} + +//////////////////////////////////////////////////////////////////////////////// +//! Time in msec. after start. If the stop watch is still running (i.e. there +//! was no call to stop()) then the elapsed time is returned added to the +//! current diff_time sum, otherwise the current summed time difference alone +//! is returned. +//////////////////////////////////////////////////////////////////////////////// +inline float StopWatchLinux::getTime() { + // Return the TOTAL time to date + float retval = total_time; + + if (running) { + retval += getDiffTime(); + } + + return retval; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Time in msec. for a single run based on the total number of COMPLETED runs +//! and the total time. +//////////////////////////////////////////////////////////////////////////////// +inline float StopWatchLinux::getAverageTime() { + return (clock_sessions > 0) ? (total_time / clock_sessions) : 0.0f; +} +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +inline float StopWatchLinux::getDiffTime() { + struct timeval t_time; + gettimeofday(&t_time, 0); + + // time difference in milli-seconds + return static_cast(1000.0 * (t_time.tv_sec - start_time.tv_sec) + + (0.001 * (t_time.tv_usec - start_time.tv_usec))); +} +#endif // WIN32 + +//////////////////////////////////////////////////////////////////////////////// +//! Timer functionality exported + +//////////////////////////////////////////////////////////////////////////////// +//! Create a new timer +//! @return true if a time has been created, otherwise false +//! @param name of the new timer, 0 if the creation failed +//////////////////////////////////////////////////////////////////////////////// +inline bool sdkCreateTimer(StopWatchInterface **timer_interface) { +// printf("sdkCreateTimer called object %08x\n", (void *)*timer_interface); +#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) + *timer_interface = reinterpret_cast(new StopWatchWin()); +#else + *timer_interface = + reinterpret_cast(new StopWatchLinux()); +#endif + return (*timer_interface != NULL) ? true : false; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Delete a timer +//! @return true if a time has been deleted, otherwise false +//! @param name of the timer to delete +//////////////////////////////////////////////////////////////////////////////// +inline bool sdkDeleteTimer(StopWatchInterface **timer_interface) { + // printf("sdkDeleteTimer called object %08x\n", (void *)*timer_interface); + if (*timer_interface) { + delete *timer_interface; + *timer_interface = NULL; + } + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Start the time with name \a name +//! @param name name of the timer to start +//////////////////////////////////////////////////////////////////////////////// +inline bool sdkStartTimer(StopWatchInterface **timer_interface) { + // printf("sdkStartTimer called object %08x\n", (void *)*timer_interface); + if (*timer_interface) { + (*timer_interface)->start(); + } + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Stop the time with name \a name. Does not reset. +//! @param name name of the timer to stop +//////////////////////////////////////////////////////////////////////////////// +inline bool sdkStopTimer(StopWatchInterface **timer_interface) { + // printf("sdkStopTimer called object %08x\n", (void *)*timer_interface); + if (*timer_interface) { + (*timer_interface)->stop(); + } + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Resets the timer's counter. +//! @param name name of the timer to reset. +//////////////////////////////////////////////////////////////////////////////// +inline bool sdkResetTimer(StopWatchInterface **timer_interface) { + // printf("sdkResetTimer called object %08x\n", (void *)*timer_interface); + if (*timer_interface) { + (*timer_interface)->reset(); + } + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// +//! Return the average time for timer execution as the total time +//! for the timer dividied by the number of completed (stopped) runs the timer +//! has made. +//! Excludes the current running time if the timer is currently running. +//! @param name name of the timer to return the time of +//////////////////////////////////////////////////////////////////////////////// +inline float sdkGetAverageTimerValue(StopWatchInterface **timer_interface) { + // printf("sdkGetAverageTimerValue called object %08x\n", (void + // *)*timer_interface); + if (*timer_interface) { + return (*timer_interface)->getAverageTime(); + } else { + return 0.0f; + } +} + +//////////////////////////////////////////////////////////////////////////////// +//! Total execution time for the timer over all runs since the last reset +//! or timer creation. +//! @param name name of the timer to obtain the value of. +//////////////////////////////////////////////////////////////////////////////// +inline float sdkGetTimerValue(StopWatchInterface **timer_interface) { + // printf("sdkGetTimerValue called object %08x\n", (void *)*timer_interface); + if (*timer_interface) { + return (*timer_interface)->getTime(); + } else { + return 0.0f; + } +} + +#endif // COMMON_HELPER_TIMER_H_ diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/include/nccl_util.h b/GPU-Virtual-Service/gpu-remoting/src/sample/include/nccl_util.h new file mode 100644 index 0000000..9134c2f --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/include/nccl_util.h @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define USE_CONFIG +#define SERVER_IP "192.168.0.208" +#define SERVER_PORT 15916 +#define BUFFER_SIZE 8 +#define GPUS_NUM 2 +#define CLIENTS_NUM 1 + +#define CUDACHECK(cmd) \ + do { \ + cudaError_t e = cmd; \ + if (e != cudaSuccess) { \ + printf("Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \ + cudaGetErrorString(e)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define NCCLCHECK(cmd) \ + do { \ + ncclResult_t r = cmd; \ + if (r != ncclSuccess) { \ + printf("Failed, NCCL error %s:%d '%s'\n", __FILE__, __LINE__, \ + ncclGetErrorString(r)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +static void PrintBuffer(void** devBuff) { + float* hostBuff = (float*)malloc(BUFFER_SIZE * sizeof(float)); + for (int devID = 0; devID < GPUS_NUM; devID++) { + printf("GPU[%d]", devID); + CUDACHECK(cudaSetDevice(devID)); + CUDACHECK(cudaMemcpy(hostBuff, devBuff[devID], BUFFER_SIZE * sizeof(float), cudaMemcpyDeviceToHost)); + + for(int i = 0; i < BUFFER_SIZE; i++) { + printf("%f ", hostBuff[i]); + } + printf("\n"); + } + free(hostBuff); +} + +static void PrintBuffer(void** devBuff, size_t cnt) { + float* hostBuff = (float*)malloc(cnt * sizeof(float)); + for (int devID = 0; devID < GPUS_NUM; devID++) { + printf("GPU[%d]", devID); + CUDACHECK(cudaSetDevice(devID)); + CUDACHECK(cudaMemcpy(hostBuff, devBuff[devID], cnt * sizeof(float), cudaMemcpyDeviceToHost)); + + for(int i = 0; i < cnt; i++) { + printf("%f ", hostBuff[i]); + } + printf("\n"); + } + free(hostBuff); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNN.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNN.cu new file mode 100644 index 0000000..08a9e9a --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNN.cu @@ -0,0 +1,339 @@ +/** +* Initialized CUDA and cuDNN context. +* Input data initialized: +* 0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 +* Input Data prepared: tensors allocated and initialized. +* Extra parameters and workspace for batch normalization allocated. +* Forward and backward propagation completed. +* Batch Normalization Output: +* -1.62621 -1.62619 -1.62617 -1.62615 -1.62613 -1.62611 -1.6261 -1.62608 -1.62606 -1.62604 +* Input Gradient: +* 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 +*/ +#include +#include +#include +#include + +#define checkCUDNN(expression) \ +{ \ + cudnnStatus_t status = (expression); \ + if (status != CUDNN_STATUS_SUCCESS) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudnnGetErrorString(status) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +#define checkCUDA(expression) \ +{ \ + cudaError_t error = (expression); \ + if (error != cudaSuccess) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudaGetErrorString(error) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +// CUDA内核，用于初始化输入数据为等差数列 +__global__ void initArithmeticSequence(float *data, int n, float start, float step) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx < n) { + data[idx] = start + step * idx; + } +} + +int main() { + cudnnHandle_t cudnn; + checkCUDNN(cudnnCreate(&cudnn)); // 初始化cuDNN + + cudaStream_t stream; + checkCUDA(cudaStreamCreate(&stream)); // 初始化CUDA流 + checkCUDNN(cudnnSetStream(cudnn, stream)); // 设置cuDNN流 + + // 数据维度和批量标准化参数 + const int batch_size = 64; // 批大小 + const int channels = 3; // 输入通道数 + const int height = 32; // 图像高度 + const int width = 32; // 图像宽度 + const float epsilon = 1e-5f; // 用于批量标准化的epsilon值 + const float momentum = 0.9f; // 用于移动平均的动量值 + + std::cout << "Initialized CUDA and cuDNN context." << std::endl; + + // 设定输入输出张量的维度 + const int tensorDims = 4; + int tensorDimsA[tensorDims] = {batch_size, channels, height, width}; // 定义维度大小 + int strideA[tensorDims] = {channels * height * width, height * width, width, 1}; // 定义步长 + + // 创建张量描述符 + cudnnTensorDescriptor_t inputDescriptor, outputDescriptor, dyDescriptor, dxDescriptor; + checkCUDNN(cudnnCreateTensorDescriptor(&inputDescriptor)); + checkCUDNN(cudnnCreateTensorDescriptor(&outputDescriptor)); + checkCUDNN(cudnnCreateTensorDescriptor(&dyDescriptor)); // 输出梯度描述符 + checkCUDNN(cudnnCreateTensorDescriptor(&dxDescriptor)); // 输入梯度描述符 + + // 设置张量描述符的格式和维度 + checkCUDNN(cudnnSetTensorNdDescriptor(inputDescriptor, + CUDNN_DATA_FLOAT, + tensorDims, + tensorDimsA, + strideA)); + // checkCUDNN(cudnnSetTensor4dDescriptor(outputDescriptor, + // CUDNN_TENSOR_NCHW, + // CUDNN_DATA_FLOAT, + // batch_size, + // channels, + // height, + // width)); + checkCUDNN(cudnnSetTensorNdDescriptorEx(outputDescriptor, + CUDNN_TENSOR_NCHW, + CUDNN_DATA_FLOAT, + tensorDims, + tensorDimsA)); + checkCUDNN(cudnnSetTensor4dDescriptor(dyDescriptor, + CUDNN_TENSOR_NCHW, + CUDNN_DATA_FLOAT, + batch_size, + channels, + height, + width)); + checkCUDNN(cudnnSetTensor4dDescriptor(dxDescriptor, + CUDNN_TENSOR_NCHW, + CUDNN_DATA_FLOAT, + batch_size, + channels, + height, + width)); + + // 为输入数据、输出数据、批量标准化参数分配GPU内存 + float *d_input, *d_output, *d_output_grad, *d_input_grad; + float *d_scale, *d_bias; + + checkCUDA(cudaMalloc(&d_input, sizeof(float) * batch_size * channels * height * width)); + checkCUDA(cudaMalloc(&d_output, sizeof(float) * batch_size * channels * height * width)); + checkCUDA(cudaMalloc(&d_output_grad, sizeof(float) * batch_size * channels * height * width)); // 输出梯度 + checkCUDA(cudaMalloc(&d_input_grad, sizeof(float) * batch_size * channels * height * width)); // 输入梯度 + checkCUDA(cudaMalloc(&d_scale, sizeof(float) * channels)); + checkCUDA(cudaMalloc(&d_bias, sizeof(float) * channels)); + + // 初始化输入数据为等差数列 + int n = batch_size * channels * height * width; + float start = 0.0f; // 等差数列起始值 + float step = 0.1f; // 等差数列步长 + int threadsPerBlock = 256; + int blocksPerGrid = (n + threadsPerBlock - 1) / threadsPerBlock; + initArithmeticSequence<<>>(d_input, n, start, step); + cudaDeviceSynchronize(); // 确保CUDA内核执行完成 + float *h_input = new float[n]; + checkCUDA(cudaMemcpy(h_input, d_input, sizeof(float) * n, cudaMemcpyDeviceToHost)); + std::cout << "Input data initialized: " << std::endl; + for (int i = 0; i < 10; i++) { + std::cout << h_input[i] << " "; + } + std::cout << std::endl; + + // 初始化批量标准化参数 + std::vector h_scale(channels, 1.0f); // scale初始化为1 + std::vector h_bias(channels, 0.0f); // bias初始化为0 + + // 将初始化的参数从主机拷贝到设备 + checkCUDA(cudaMemcpy(d_scale, h_scale.data(), sizeof(float) * channels, cudaMemcpyHostToDevice)); + checkCUDA(cudaMemcpy(d_bias, h_bias.data(), sizeof(float) * channels, cudaMemcpyHostToDevice)); + + std::cout << "Input Data prepared: tensors allocated and initialized." << std::endl; + + // 前向传播和反向传播的额外参数 + float *d_runningMean, *d_runningVariance; + float *d_saveMean, *d_saveInvVariance; // 前向训练时保存的均值和逆方差 + float *d_bnScaleDiff, *d_bnBiasDiff; // 反向传播时计算的梯度 + void *d_workspace = nullptr; // 反向传播所需的工作空间 + size_t workspaceSize = 0; // 工作空间的大小 + size_t reserveSpaceSize = 0; // 保留空间的大小 + + // 分配额外参数的内存 + checkCUDA(cudaMalloc(&d_runningMean, sizeof(float) * channels)); + checkCUDA(cudaMalloc(&d_runningVariance, sizeof(float) * channels)); + checkCUDA(cudaMalloc(&d_saveMean, sizeof(float) * channels)); + checkCUDA(cudaMalloc(&d_saveInvVariance, sizeof(float) * channels)); + checkCUDA(cudaMalloc(&d_bnScaleDiff, sizeof(float) * channels)); + checkCUDA(cudaMalloc(&d_bnBiasDiff, sizeof(float) * channels)); + + // 初始化为0 + checkCUDA(cudaMemset(d_runningMean, 0, sizeof(float) * channels)); + checkCUDA(cudaMemset(d_runningVariance, 0, sizeof(float) * channels)); + + // 获取前向传播和反向传播所需的工作空间大小 + cudnnTensorDescriptor_t bnScaleBiasMeanVarDesc; + checkCUDNN(cudnnCreateTensorDescriptor(&bnScaleBiasMeanVarDesc)); + checkCUDNN(cudnnSetTensor4dDescriptor(bnScaleBiasMeanVarDesc, + CUDNN_TENSOR_NCHW, + CUDNN_DATA_FLOAT, + 1, channels, 1, 1)); + + cudnnBatchNormMode_t mode = CUDNN_BATCHNORM_SPATIAL_PERSISTENT; + cudnnBatchNormOps_t bnOps = CUDNN_BATCHNORM_OPS_BN; + // 假设与批量标准化操作不结合使用任何额外的激活函数 + cudnnActivationDescriptor_t activationDesc = nullptr; + + // 为前向训练计算工作空间大小 + checkCUDNN(cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize( + cudnn, // cudnn句柄 + mode, // 批量标准化模式 + bnOps, // 批量标准化操作 + inputDescriptor, // xDesc: 输入张量描述符 + nullptr, // zDesc: 偏置张量描述符（在这里不使用，因此为nullptr） + outputDescriptor, // yDesc: 输出张量描述符 + bnScaleBiasMeanVarDesc, // bnScaleBiasMeanVarDesc: 缩放、偏置、均值和方差的描述符 + activationDesc, // activationDesc: 激活描述符（在这里不使用，因此为nullptr） + &workspaceSize // sizeInBytes: 工作空间大小的指针 + )); + + // 为训练计算保留空间大小 + checkCUDNN(cudnnGetBatchNormalizationTrainingExReserveSpaceSize( + cudnn, // cudnn句柄 + mode, // 批量标准化模式 + bnOps, // 批量标准化操作 + activationDesc, // activationDesc: 激活描述符（在这里不使用，因此为nullptr） + inputDescriptor, // xDesc: 输入张量描述符 + &reserveSpaceSize // sizeInBytes: 保留空间大小的指针 + )); + + // 分配工作空间和保留空间 + void *d_reserveSpace = nullptr; // 用于反向传播的保留空间 + if (workspaceSize > 0) { + checkCUDA(cudaMalloc(&d_workspace, workspaceSize)); + std::cout << "Workspace for batch normalization allocated: " << workspaceSize << " bytes." << std::endl; + } + if (reserveSpaceSize > 0) { + checkCUDA(cudaMalloc(&d_reserveSpace, reserveSpaceSize)); + std::cout << "Reserve space for batch normalization allocated: " << reserveSpaceSize << " bytes." << std::endl; + } + + std::cout << "Extra parameters and workspace for batch normalization allocated." << std::endl; + + // 设置学习率参数，这些在实际训练中可能会变化或根据优化算法调整 + const float alpha = 1.0f; + const float beta = 0.0f; + + // 前向训练 + checkCUDNN(cudnnBatchNormalizationForwardTrainingEx( + cudnn, mode, bnOps, + &alpha, &beta, inputDescriptor, d_input, nullptr, nullptr, + outputDescriptor, d_output, bnScaleBiasMeanVarDesc, + d_scale, d_bias, momentum, d_runningMean, d_runningVariance, + epsilon, d_saveMean, d_saveInvVariance, activationDesc, + d_workspace, workspaceSize, d_reserveSpace, reserveSpaceSize + )); + + // 前向推理（用于评估或推理） + // 使用训练过程中计算得到的运行均值和方差 + checkCUDNN(cudnnBatchNormalizationForwardInference( + cudnn, mode, + &alpha, &beta, inputDescriptor, d_input, + outputDescriptor, d_output, bnScaleBiasMeanVarDesc, + d_scale, d_bias, d_runningMean, d_runningVariance, epsilon + )); + + // 执行批量标准化的反向传播 + const float alphaDataDiff = 1.0f; // 输出梯度缩放系数 + const float betaDataDiff = 0.0f; + const float alphaParamDiff = 1.0f; // 参数梯度缩放系数 + const float betaParamDiff = 0.0f; + const float outputGradVal = 1.0f; + checkCUDA(cudaMemset(d_output_grad, outputGradVal, sizeof(float) * batch_size * channels * height * width)); // 输出梯度初始化为1 + + size_t workspaceSizeBackward = 0; // 反向传播所需的工作空间大小 + checkCUDNN(cudnnGetBatchNormalizationBackwardExWorkspaceSize( + cudnn, // cudnn句柄 + mode, // 批量标准化模式 + bnOps, // 批量标准化操作 + inputDescriptor, // xDesc: 输入张量描述符 + outputDescriptor, // yDesc: 输出张量描述符 + dyDescriptor, // dyDesc: 输出梯度张量描述符 + nullptr, // dzDesc: 输入梯度张量描述符（在这里不使用，因此为nullptr） + dxDescriptor, // dxDesc: 输入梯度张量描述符 + bnScaleBiasMeanVarDesc, // bnScaleBiasMeanVarDesc: 缩放、偏置、均值和方差的描述符 + activationDesc, // activationDesc: 激活描述符（如果使用激活函数的话） + &workspaceSizeBackward // sizeInBytes: 指向存储工作空间大小的变量的指针 + )); + if (workspaceSizeBackward > 0) { + if (workspaceSizeBackward > workspaceSize) { + // 如果反向传播需要的工作空间比前向传播更大，重新分配 + cudaFree(d_workspace); // 释放之前分配的工作空间 + checkCUDA(cudaMalloc(&d_workspace, workspaceSizeBackward)); + } + // 如果前向传播的工作空间已经足够大，则无需重新分配 + std::cout << "Workspace for batch normalization backward allocated: " << workspaceSizeBackward << " bytes." << std::endl; + } + + checkCUDNN(cudnnBatchNormalizationBackwardEx( + cudnn, mode, bnOps, + &alphaDataDiff, &betaDataDiff, &alphaParamDiff, &betaParamDiff, + inputDescriptor, d_input, outputDescriptor, d_output, + dyDescriptor, d_output_grad, nullptr, nullptr, dxDescriptor, d_input_grad, + bnScaleBiasMeanVarDesc, d_scale, d_bias, d_bnScaleDiff, d_bnBiasDiff, + epsilon, d_saveMean, d_saveInvVariance, activationDesc, + d_workspace, workspaceSize, d_reserveSpace, reserveSpaceSize + )); + + std::cout << "Forward and backward propagation completed." << std::endl; + + // 将输出数据从GPU拷贝回主机 + float *h_output; + h_output = new float[batch_size * channels * height * width]; + checkCUDA(cudaMemcpy(h_output, d_output, sizeof(float) * batch_size * channels * height * width, cudaMemcpyDeviceToHost)); + + // 将输入梯度从GPU拷贝回主机 + float *h_input_grad; + h_input_grad = new float[batch_size * channels * height * width]; + checkCUDA(cudaMemcpy(h_input_grad, d_input_grad, sizeof(float) * batch_size * channels * height * width, cudaMemcpyDeviceToHost)); + + // 输出批量标准化的结果 + std::cout << "Batch Normalization Output:" << std::endl; + for (int i = 0; i < 10; i++) { + std::cout << h_output[i] << " "; + } + std::cout << std::endl; + // -1.62621 -1.62619 -1.62617 -1.62615 -1.62613 -1.62611 -1.6261 -1.62608 -1.62606 -1.6260 + + // 输出输入梯度 + std::cout << "Input Gradient:" << std::endl; + for (int i = 0; i < 10; i++) { + std::cout << h_input_grad[i] << " "; + } + std::cout << std::endl; + // 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 4.17587e-42 + + // 释放主机内存 + delete[] h_output; + delete[] h_input_grad; + + // 同步GPU并释放内存 + cudaDeviceSynchronize(); + cudaFree(d_input); + cudaFree(d_output); + cudaFree(d_output_grad); + cudaFree(d_input_grad); + cudaFree(d_scale); + cudaFree(d_bias); + cudaFree(d_runningMean); + cudaFree(d_runningVariance); + cudaFree(d_saveMean); + cudaFree(d_saveInvVariance); + cudaFree(d_bnScaleDiff); + cudaFree(d_bnBiasDiff); + cudaFree(d_workspace); + cudaFree(d_reserveSpace); + + // 清理cuDNN资源 + cudnnDestroyTensorDescriptor(inputDescriptor); + cudnnDestroyTensorDescriptor(outputDescriptor); + cudnnDestroyTensorDescriptor(dyDescriptor); + cudnnDestroyTensorDescriptor(dxDescriptor); + cudnnDestroyTensorDescriptor(bnScaleBiasMeanVarDesc); + cudnnDestroy(cudnn); + cudaStreamDestroy(stream); + + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNBackend.cu.bak b/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNBackend.cu.bak new file mode 100644 index 0000000..93e4e94 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNBackend.cu.bak @@ -0,0 +1,184 @@ +#include +#include +#include + +#define checkCUDNN(expression) \ +{ \ + cudnnStatus_t status = (expression); \ + if (status != CUDNN_STATUS_SUCCESS) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudnnGetErrorString(status) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +#define checkCUDA(expression) \ +{ \ + cudaError_t error = (expression); \ + if (error != cudaSuccess) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudaGetErrorString(error) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +int main() { + cudnnHandle_t handle; + checkCUDNN(cudnnCreate(&handle)); + + // Create a input tensor descriptors, convolution filter and output tensor descriptor + + cudnnBackendDescriptor_t xDesc; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_TENSOR_DESCRIPTOR, &xDesc)); + cudnnDataType_t dtype = CUDNN_DATA_FLOAT; + checkCUDNN(cudnnBackendSetAttribute(xDesc, CUDNN_ATTR_TENSOR_DATA_TYPE, CUDNN_TYPE_DATA_TYPE, 1, &dtype)); + // [n, g, c, d, h, w] = [32, 4, 16, 8, 28, 28] + int64_t n = 32, g = 4, c = 16, d = 8, h = 28, w = 28; + int64_t xDim[] = {n, g, c, d, h, w}; + int64_t xStr[] = {g * c * d * h * w, c * d * h * w, d * h * w, h * w, w, 1}; + int64_t xUi = 'x'; + int64_t alignment = 4; + checkCUDNN(cudnnBackendSetAttribute(xDesc, CUDNN_ATTR_TENSOR_DIMENSIONS, CUDNN_TYPE_INT64, 6, xDim)); + checkCUDNN(cudnnBackendSetAttribute(xDesc, CUDNN_ATTR_TENSOR_STRIDES, CUDNN_TYPE_INT64, 6, xStr)); + checkCUDNN(cudnnBackendSetAttribute(xDesc, CUDNN_ATTR_TENSOR_UNIQUE_ID, CUDNN_TYPE_INT64, 1, &xUi)); + checkCUDNN(cudnnBackendSetAttribute(xDesc, CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT, CUDNN_TYPE_INT64, 1, &alignment)); + checkCUDNN(cudnnBackendFinalize(xDesc)); + + cudnnBackendDescriptor_t wDesc; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_TENSOR_DESCRIPTOR, &wDesc)); + checkCUDNN(cudnnBackendSetAttribute(wDesc, CUDNN_ATTR_TENSOR_DATA_TYPE, CUDNN_TYPE_DATA_TYPE, 1, &dtype)); + // [g, k, c, t, r, s] = [4, 32, 16, 4, 4, 4] + int64_t k = 32, t = 4, r = 4, s = 4; + int64_t wDim[] = {g, k, c, t, r, s}; + int64_t wStr[] = {k * c * t * r * s, c * t * r * s, t * r * s, r * s, s, 1}; + int64_t wUi = 'w'; + checkCUDNN(cudnnBackendSetAttribute(wDesc, CUDNN_ATTR_TENSOR_DIMENSIONS, CUDNN_TYPE_INT64, 6, wDim)); + checkCUDNN(cudnnBackendSetAttribute(wDesc, CUDNN_ATTR_TENSOR_STRIDES, CUDNN_TYPE_INT64, 6, wStr)); + checkCUDNN(cudnnBackendSetAttribute(wDesc, CUDNN_ATTR_TENSOR_UNIQUE_ID, CUDNN_TYPE_INT64, 1, &wUi)); + checkCUDNN(cudnnBackendSetAttribute(wDesc, CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT, CUDNN_TYPE_INT64, 1, &alignment)); + checkCUDNN(cudnnBackendFinalize(wDesc)); + + cudnnBackendDescriptor_t yDesc; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_TENSOR_DESCRIPTOR, &yDesc)); + checkCUDNN(cudnnBackendSetAttribute(yDesc, CUDNN_ATTR_TENSOR_DATA_TYPE, CUDNN_TYPE_DATA_TYPE, 1, &dtype)); + //[n, g, k, o, p, q] = [32, 4, 32, 4, 25, 25] + int64_t o = 4, p = 25, q = 25; + int64_t yDim[] = {n, g, k, o, p, q}; + int64_t yStr[] = {g * k * o * p * q, k * o * p * q, o * p * q, p * q, q, 1}; + int64_t yUi = 'y'; + checkCUDNN(cudnnBackendSetAttribute(yDesc, CUDNN_ATTR_TENSOR_DIMENSIONS, CUDNN_TYPE_INT64, 6, yDim)); + checkCUDNN(cudnnBackendSetAttribute(yDesc, CUDNN_ATTR_TENSOR_STRIDES, CUDNN_TYPE_INT64, 6, yStr)); + checkCUDNN(cudnnBackendSetAttribute(yDesc, CUDNN_ATTR_TENSOR_UNIQUE_ID, CUDNN_TYPE_INT64, 1, &yUi)); + checkCUDNN(cudnnBackendSetAttribute(yDesc, CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT, CUDNN_TYPE_INT64, 1, &alignment)); + checkCUDNN(cudnnBackendFinalize(yDesc)); + + + // Create, set, and finalize a convolution operator descriptor. + + cudnnBackendDescriptor_t cDesc; + int64_t nbDims = 3; + cudnnDataType_t compType = CUDNN_DATA_FLOAT; + cudnnConvolutionMode_t mode = CUDNN_CONVOLUTION; + int64_t pad[] = {0, 0, 0}; + int64_t filterStr[] = {1, 1, 1}; + int64_t dilation[] = {1, 1, 1}; + + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR, &cDesc)); + + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS, CUDNN_TYPE_INT64, 1, &nbDims)); + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_COMP_TYPE, CUDNN_TYPE_DATA_TYPE, 1, &compType)); + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_CONV_MODE, CUDNN_TYPE_CONVOLUTION_MODE, 1, &mode)); + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS, CUDNN_TYPE_INT64, nbDims, pad)); + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_POST_PADDINGS, CUDNN_TYPE_INT64, nbDims, pad)); + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_DILATIONS, CUDNN_TYPE_INT64, nbDims, dilation)); + checkCUDNN(cudnnBackendSetAttribute(cDesc, CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES, CUDNN_TYPE_INT64, nbDims, filterStr)); + checkCUDNN(cudnnBackendFinalize(cDesc)); + + //Create, set, and finalize a convolution forward operation descriptor. + + cudnnBackendDescriptor_t fprop; + float alpha = 1.0; + float beta = 0.5; + + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR, &fprop)); + checkCUDNN(cudnnBackendSetAttribute(fprop, CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_X, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &xDesc)); + checkCUDNN(cudnnBackendSetAttribute(fprop, CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_W, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &wDesc)); + checkCUDNN(cudnnBackendSetAttribute(fprop, CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_Y, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &yDesc)); + checkCUDNN(cudnnBackendSetAttribute(fprop, CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_CONV_DESC, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &cDesc)); + checkCUDNN(cudnnBackendSetAttribute(fprop, CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_ALPHA, CUDNN_TYPE_FLOAT, 1, &alpha)); + checkCUDNN(cudnnBackendSetAttribute(fprop, CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_BETA, CUDNN_TYPE_FLOAT, 1, &beta)); + checkCUDNN(cudnnBackendFinalize(fprop)); + + //Create, set, and finalize an operation graph descriptor. + + cudnnBackendDescriptor_t op_graph; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATIONGRAPH_DESCRIPTOR, &op_graph)); + checkCUDNN(cudnnBackendSetAttribute(op_graph, CUDNN_ATTR_OPERATIONGRAPH_OPS, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &fprop)); + checkCUDNN(cudnnBackendSetAttribute(op_graph, CUDNN_ATTR_OPERATIONGRAPH_HANDLE, CUDNN_TYPE_HANDLE, 1, &handle)); + checkCUDNN(cudnnBackendFinalize(op_graph)); + + //Create, set, and finalize an engine descriptor. + + cudnnBackendDescriptor_t engine; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_ENGINE_DESCRIPTOR, &engine)); + checkCUDNN(cudnnBackendSetAttribute(engine, CUDNN_ATTR_ENGINE_OPERATION_GRAPH, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &op_graph)); + int64_t gidx = 0; + checkCUDNN(cudnnBackendSetAttribute(engine, CUDNN_ATTR_ENGINE_GLOBAL_INDEX, CUDNN_TYPE_INT64, 1, &gidx)); + checkCUDNN(cudnnBackendFinalize(engine)); + + //Create, set, and finalize an engine config descriptor. + + cudnnBackendDescriptor_t engcfg; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_ENGINECFG_DESCRIPTOR, &engcfg)); + checkCUDNN(cudnnBackendSetAttribute(engcfg, CUDNN_ATTR_ENGINECFG_ENGINE, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &engine)); + checkCUDNN(cudnnBackendFinalize(engcfg)); + + //Create, set, and finalize an execution plan descriptor. Obtain workspace size to allocate. + + cudnnBackendDescriptor_t plan; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR, &plan)); + checkCUDNN(cudnnBackendSetAttribute(plan, CUDNN_ATTR_EXECUTION_PLAN_HANDLE, CUDNN_TYPE_HANDLE, 1, &handle)); + checkCUDNN(cudnnBackendSetAttribute(plan, CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &engcfg)); + checkCUDNN(cudnnBackendFinalize(plan)); + + int64_t workspaceSize; + checkCUDNN(cudnnBackendGetAttribute(plan, CUDNN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE, CUDNN_TYPE_INT64, 1, NULL, &workspaceSize)); + + //Create, set and finalize a variant pack descriptor. + float* x_host = (float*)malloc(sizeof(float) * n * g * c * d * h * w); + float* w_host = (float*)malloc(sizeof(float) * g * k * c * t * r * s); + float* y_host = (float*)malloc(sizeof(float) * n * g * k * o * p * q); + for (int i = 0; i < n * g * c * d * h * w; i++) { + x_host[i] = 0.1 * i; + w_host[i] = 0.2 * i; + y_host[i] = 0.0; + } + + float *xData, *wData, *yData; + checkCUDA(cudaMalloc((void**)&xData, sizeof(float) * n * g * c * d * h * w)); + checkCUDA(cudaMalloc((void**)&wData, sizeof(float) * g * k * c * t * r * s)); + checkCUDA(cudaMalloc((void**)&yData, sizeof(float) * n * g * k * o * p * q)); + checkCUDA(cudaMemcpy(xData, x_host, sizeof(float) * n * g * c * d * h * w, cudaMemcpyHostToDevice)); + checkCUDA(cudaMemcpy(wData, w_host, sizeof(float) * g * k * c * t * r * s, cudaMemcpyHostToDevice)); + + void *dev_ptrs[3] = {xData, wData, yData}; // device pointer + int64_t uids[3] = {'x', 'w', 'y'}; + void *workspace; + + cudnnBackendDescriptor_t varpack; + checkCUDNN(cudnnBackendCreateDescriptor(CUDNN_BACKEND_VARIANT_PACK_DESCRIPTOR, &varpack)); + checkCUDNN(cudnnBackendSetAttribute(varpack, CUDNN_ATTR_VARIANT_PACK_DATA_POINTERS, CUDNN_TYPE_VOID_PTR, 3, dev_ptrs)); + checkCUDNN(cudnnBackendSetAttribute(varpack, CUDNN_ATTR_VARIANT_PACK_UNIQUE_IDS, CUDNN_TYPE_INT64, 3, uids)); + checkCUDNN(cudnnBackendSetAttribute(varpack, CUDNN_ATTR_VARIANT_PACK_WORKSPACE, CUDNN_TYPE_VOID_PTR, 1, &workspace)); + checkCUDNN(cudnnBackendFinalize(varpack)); + + //Execute the plan with a variant pack. + checkCUDNN(cudnnBackendExecute(handle, plan, varpack)); + checkCUDA(cudaMemcpy(y_host, yData, sizeof(float) * n * g * k * o * p * q, cudaMemcpyDeviceToHost)); + + cudaFree(xData); + cudaFree(wData); + cudaFree(yData); + + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNTrans.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNTrans.cu new file mode 100644 index 0000000..0329cbb --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testCuDNNTrans.cu @@ -0,0 +1,51 @@ +#include +#include + +#define checkCUDNN(expression) \ +{ \ + cudnnStatus_t status = (expression); \ + if (status != CUDNN_STATUS_SUCCESS) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudnnGetErrorString(status) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +int main() { + cudnnHandle_t cudnn; + checkCUDNN(cudnnCreate(&cudnn)); + + // Tensor Descriptor for src and dest + cudnnTensorDescriptor_t srcDesc, destDesc; + checkCUDNN(cudnnCreateTensorDescriptor(&srcDesc)); + checkCUDNN(cudnnCreateTensorDescriptor(&destDesc)); + int srcDimA[4] = {512, 256, 1, 1}; + checkCUDNN(cudnnSetTensorNdDescriptorEx(srcDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 4, srcDimA)); + + // Filter Descriptor + cudnnFilterDescriptor_t filterDesc; + checkCUDNN(cudnnCreateFilterDescriptor(&filterDesc)); + int filterDimA[4] = {512, 1024, 1, 1}; + checkCUDNN(cudnnSetFilterNdDescriptor(filterDesc, CUDNN_DATA_FLOAT, CUDNN_TENSOR_NCHW, 4, filterDimA)); + + // Tensor Transform Descriptor + cudnnTensorTransformDescriptor_t transformDesc; + checkCUDNN(cudnnCreateTensorTransformDescriptor(&transformDesc)); + int padBeforeA[4] = {0, 0, 0, 0}; + int padAfterA[4] = {0, 0, 1, 1}; + uint32_t foldA[2] = {2, 2}; // Considering spatial dimensions only as per documentation + checkCUDNN(cudnnSetTensorTransformDescriptor(transformDesc, 4, CUDNN_TENSOR_NCHW, padBeforeA, padAfterA, foldA, CUDNN_TRANSFORM_FOLD)); + + // Initialize dest tensor descriptor based on transformation + size_t destSizeInBytes; + checkCUDNN(cudnnInitTransformDest(transformDesc, srcDesc, destDesc, &destSizeInBytes)); + + // Clean up + checkCUDNN(cudnnDestroyTensorDescriptor(srcDesc)); + checkCUDNN(cudnnDestroyTensorDescriptor(destDesc)); + checkCUDNN(cudnnDestroyFilterDescriptor(filterDesc)); + checkCUDNN(cudnnDestroyTensorTransformDescriptor(transformDesc)); + cudnnDestroy(cudnn); + + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testCublas.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testCublas.cu new file mode 100644 index 0000000..1e67765 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testCublas.cu @@ -0,0 +1,63 @@ +/** +* 30 84 138 +* 24 69 114 +* 18 54 90 +*/ +#include +#include + +int main() { + cublasHandle_t handle; + const int N = 3; + + // 初始化 cuBLAS 句柄 + cublasCreate(&handle); + + // 分配和初始化矩阵 + float a[N*N] = {1.0, 2.0, 3.0, + 4.0, 5.0, 6.0, + 7.0, 8.0, 9.0}; + float b[N*N] = {9.0, 8.0, 7.0, + 6.0, 5.0, 4.0, + 3.0, 2.0, 1.0}; + float c[N*N] = {0}; + + float *d_a, *d_b, *d_c; + cudaMalloc((void **)&d_a, sizeof(a)); + cudaMalloc((void **)&d_b, sizeof(b)); + cudaMalloc((void **)&d_c, sizeof(c)); + + // 将数据复制到设备 + cudaMemcpy(d_a, a, sizeof(a), cudaMemcpyHostToDevice); + cudaMemcpy(d_b, b, sizeof(b), cudaMemcpyHostToDevice); + + size_t workspace_size = 10 * 1024 * 1024; // 10 MB + void* workspace; + cudaMalloc(&workspace, workspace_size); + cublasSetWorkspace_v2(handle, workspace, workspace_size); + + // 执行矩阵乘法 + const float alpha = 1.0f; + const float beta = 0.0f; + cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T, N, N, N, &alpha, d_a, N, d_b, N, &beta, d_c, N); + + // 将结果复制回主机 + cudaMemcpy(c, d_c, sizeof(c), cudaMemcpyDeviceToHost); + + // 打印结果 + for (int i = 0; i < N; i++) { + for (int j = 0; j < N; j++) { + std::cout << c[i * N + j] << " "; + } + std::cout << std::endl; + } + + // 清理 + cudaFree(d_a); + cudaFree(d_b); + cudaFree(d_c); + cudaFree(workspace); + cublasDestroy(handle); + + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testCublasLt.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testCublasLt.cu new file mode 100644 index 0000000..3a578b8 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testCublasLt.cu @@ -0,0 +1,125 @@ +/** +* cuBLASLt library version:111003 +* Heuristic result: +* algorithm descriptor:11,4294967296,0,476754254757889,0,0,68,80, +* Actual size of workspace memory required:0 +* Result status:Success +* Result matrix: +* 90 114 138 +* 54 69 84 +* 18 24 30 +*/ +#include +#include +#include +#include + +// CUDA 和 cuBLASLt 错误检查宏 +#define CHECK_CUDA(expression) \ +{ \ + cudaError_t error = (expression); \ + if (error != cudaSuccess) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudaGetErrorString(error) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +#define CHECK_CUBLASLT(expression) \ +{ \ + cublasStatus_t cublasLtStatus = (expression); \ + if (cublasLtStatus != CUBLAS_STATUS_SUCCESS) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cublasLtGetStatusString(cublasLtStatus) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +int main() { + const uint64_t N = 3; + float a[N*N] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0}; + float b[N*N] = {9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0}; + float c[N*N] = {0}; + + std::cout << "cuBLASLt library version:" << cublasLtGetVersion() << std::endl; + + // 创建 cuBLASLt 句柄 + cublasLtHandle_t handle; + CHECK_CUBLASLT(cublasLtCreate(&handle)); + + // 在 GPU 上分配内存 + float *d_a, *d_b, *d_c; + CHECK_CUDA(cudaMalloc((void **)&d_a, sizeof(a))); + CHECK_CUDA(cudaMalloc((void **)&d_b, sizeof(b))); + CHECK_CUDA(cudaMalloc((void **)&d_c, sizeof(c))); + + // 将数据复制到设备 + CHECK_CUDA(cudaMemcpy(d_a, a, sizeof(a), cudaMemcpyHostToDevice)); + CHECK_CUDA(cudaMemcpy(d_b, b, sizeof(b), cudaMemcpyHostToDevice)); + + // 定义矩阵乘法操作的描述 + cublasLtMatmulDesc_t operationDesc; + cublasLtMatrixLayout_t aLayout, bLayout, cLayout; + cublasLtMatmulPreference_t preference; + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult; + + CHECK_CUBLASLT(cublasLtMatmulDescCreate(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); + cublasOperation_t transa = CUBLAS_OP_N; // 设置矩阵 A 的转置类型 + CHECK_CUBLASLT(cublasLtMatmulDescSetAttribute(operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa))); + + CHECK_CUBLASLT(cublasLtMatrixLayoutCreate(&aLayout, CUDA_R_32F, N, N - 2, N)); + CHECK_CUBLASLT(cublasLtMatrixLayoutCreate(&bLayout, CUDA_R_32F, N, N, N)); + CHECK_CUBLASLT(cublasLtMatrixLayoutCreate(&cLayout, CUDA_R_32F, N, N, N)); + CHECK_CUBLASLT(cublasLtMatrixLayoutSetAttribute(aLayout, CUBLASLT_MATRIX_LAYOUT_COLS, &N, sizeof(N))); + + size_t workspaceSize = 10 * 1024 * 1024; + CHECK_CUBLASLT(cublasLtMatmulPreferenceCreate(&preference)); + CHECK_CUBLASLT(cublasLtMatmulPreferenceSetAttribute(preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize))); + + // 获取最佳算法 + CHECK_CUBLASLT(cublasLtMatmulAlgoGetHeuristic(handle, operationDesc, aLayout, bLayout, cLayout, cLayout, preference, 1, &heuristicResult, &returnedResults)); + + if (returnedResults == 0) { + std::cerr << "No suitable algorithm found!" << std::endl; + exit(-1); + } + else { + std::cout << "Heuristic result:" << std::endl; + std::cout << "algorithm descriptor:"; + for (int i = 0; i < 8; i ++) { + std::cout << heuristicResult.algo.data[i] << ","; + } + std::cout << std::endl << "Actual size of workspace memory required:" << heuristicResult.workspaceSize << std::endl; + std::cout << "Result status:" << (heuristicResult.state == CUBLAS_STATUS_SUCCESS ? "Success" : "Failed") << std::endl; + } + + // 执行矩阵乘法 + const float alpha = 1.0f, beta = 0.0f; + CHECK_CUBLASLT(cublasLtMatmul(handle, operationDesc, &alpha, d_a, aLayout, d_b, bLayout, &beta, d_c, cLayout, d_c, cLayout, &heuristicResult.algo, nullptr, 0, 0)); + + // 将结果复制回主机 + CHECK_CUDA(cudaMemcpy(c, d_c, sizeof(c), cudaMemcpyDeviceToHost)); + + // 打印结果 + std::cout << "Result matrix:" << std::endl; + for (int i = 0; i < N; i++) { + for (int j = 0; j < N; j++) { + std::cout << c[i * N + j] << " "; + } + std::cout << std::endl; + } + + // 清理资源 + cublasLtMatrixLayoutDestroy(aLayout); + cublasLtMatrixLayoutDestroy(bLayout); + cublasLtMatrixLayoutDestroy(cLayout); + cublasLtMatmulDescDestroy(operationDesc); + cublasLtMatmulPreferenceDestroy(preference); + cudaFree(d_a); + cudaFree(d_b); + cudaFree(d_c); + cublasLtDestroy(handle); + + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testDevice.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testDevice.cu new file mode 100644 index 0000000..9a25080 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testDevice.cu @@ -0,0 +1,165 @@ +#include +#include +#include +#include +#include + +void compareDeviceProps(const cudaDeviceProp& prop1, const cudaDeviceProp& prop2) { + + for (size_t i = 0; i < sizeof(cudaDeviceProp); i++) { + if (((uint8_t*)&prop1)[i] != ((uint8_t*)&prop2)[i]) { + printf("prop1[%02lu]: %02x, prop2[%02lu]: %02x", i, ((uint8_t*)&prop1)[i], i, ((uint8_t*)&prop2)[i]); + printf(" <- Different\n"); + } + } + + if (prop1.major != prop2.major) + std::cout << "Different 'major' version: " << prop1.major << " vs " << prop2.major << std::endl; + if (prop1.minor != prop2.minor) + std::cout << "Different 'minor' version: " << prop1.minor << " vs " << prop2.minor << std::endl; + if (memcmp(prop1.name, prop2.name, sizeof(prop1.name)) != 0) + std::cout << "Different 'name': " << prop1.name << " vs " << prop2.name << std::endl; + if (prop1.totalGlobalMem != prop2.totalGlobalMem) + std::cout << "Different 'totalGlobalMem': " << prop1.totalGlobalMem << " vs " << prop2.totalGlobalMem << std::endl; + if (prop1.sharedMemPerBlock != prop2.sharedMemPerBlock) + std::cout << "Different 'sharedMemPerBlock': " << prop1.sharedMemPerBlock << " vs " << prop2.sharedMemPerBlock << std::endl; + if (prop1.regsPerBlock != prop2.regsPerBlock) + std::cout << "Different 'regsPerBlock': " << prop1.regsPerBlock << " vs " << prop2.regsPerBlock << std::endl; + if (prop1.warpSize != prop2.warpSize) + std::cout << "Different 'warpSize': " << prop1.warpSize << " vs " << prop2.warpSize << std::endl; + if (prop1.memPitch != prop2.memPitch) + std::cout << "Different 'memPitch': " << prop1.memPitch << " vs " << prop2.memPitch << std::endl; + if (prop1.maxThreadsPerBlock != prop2.maxThreadsPerBlock) + std::cout << "Different 'maxThreadsPerBlock': " << prop1.maxThreadsPerBlock << " vs " << prop2.maxThreadsPerBlock << std::endl; + if (memcmp(prop1.maxThreadsDim, prop2.maxThreadsDim, sizeof(prop1.maxThreadsDim)) != 0) + std::cout << "Different 'maxThreadsDim': " << prop1.maxThreadsDim[0] << ", " << prop1.maxThreadsDim[1] << ", " << prop1.maxThreadsDim[2] + << " vs " << prop2.maxThreadsDim[0] << ", " << prop2.maxThreadsDim[1] << ", " << prop2.maxThreadsDim[2] << std::endl; + if (memcmp(prop1.maxGridSize, prop2.maxGridSize, sizeof(prop1.maxGridSize)) != 0) + std::cout << "Different 'maxGridSize': " << prop1.maxGridSize[0] << ", " << prop1.maxGridSize[1] << ", " << prop1.maxGridSize[2] + << " vs " << prop2.maxGridSize[0] << ", " << prop2.maxGridSize[1] << ", " << prop2.maxGridSize[2] << std::endl; + if (prop1.clockRate != prop2.clockRate) + std::cout << "Different 'clockRate': " << prop1.clockRate << " vs " << prop2.clockRate << std::endl; + if (prop1.totalConstMem != prop2.totalConstMem) + std::cout << "Different 'totalConstMem': " << prop1.totalConstMem << " vs " << prop2.totalConstMem << std::endl; + if (prop1.multiProcessorCount != prop2.multiProcessorCount) + std::cout << "Different 'multiProcessorCount': " << prop1.multiProcessorCount << " vs " << prop2.multiProcessorCount << std::endl; + if (prop1.kernelExecTimeoutEnabled != prop2.kernelExecTimeoutEnabled) + std::cout << "Different 'kernelExecTimeoutEnabled': " << prop1.kernelExecTimeoutEnabled << " vs " << prop2.kernelExecTimeoutEnabled << std::endl; + if (prop1.integrated != prop2.integrated) + std::cout << "Different 'integrated': " << prop1.integrated << " vs " << prop2.integrated << std::endl; + if (prop1.canMapHostMemory != prop2.canMapHostMemory) + std::cout << "Different 'canMapHostMemory': " << prop1.canMapHostMemory << " vs " << prop2.canMapHostMemory << std::endl; + if (prop1.computeMode != prop2.computeMode) + std::cout << "Different 'computeMode': " << prop1.computeMode << " vs " << prop2.computeMode << std::endl; + if (prop1.concurrentKernels != prop2.concurrentKernels) + std::cout << "Different 'concurrentKernels': " << prop1.concurrentKernels << " vs " << prop2.concurrentKernels << std::endl; + if (prop1.ECCEnabled != prop2.ECCEnabled) + std::cout << "Different 'ECCEnabled': " << prop1.ECCEnabled << " vs " << prop2.ECCEnabled << std::endl; + if (prop1.pciBusID != prop2.pciBusID) + std::cout << "Different 'pciBusID': " << prop1.pciBusID << " vs " << prop2.pciBusID << std::endl; + if (prop1.pciDeviceID != prop2.pciDeviceID) + std::cout << "Different 'pciDeviceID': " << prop1.pciDeviceID << " vs " << prop2.pciDeviceID << std::endl; + if (prop1.pciDomainID != prop2.pciDomainID) + std::cout << "Different 'pciDomainID': " << prop1.pciDomainID << " vs " << prop2.pciDomainID << std::endl; + if (prop1.tccDriver != prop2.tccDriver) + std::cout << "Different 'tccDriver': " << prop1.tccDriver << " vs " << prop2.tccDriver << std::endl; + if (prop1.asyncEngineCount != prop2.asyncEngineCount) + std::cout << "Different 'asyncEngineCount': " << prop1.asyncEngineCount << " vs " << prop2.asyncEngineCount << std::endl; + if (prop1.unifiedAddressing != prop2.unifiedAddressing) + std::cout << "Different 'unifiedAddressing': " << prop1.unifiedAddressing << " vs " << prop2.unifiedAddressing << std::endl; + if (prop1.memoryClockRate != prop2.memoryClockRate) + std::cout << "Different 'memoryClockRate': " << prop1.memoryClockRate << " vs " << prop2.memoryClockRate << std::endl; + if (prop1.memoryBusWidth != prop2.memoryBusWidth) + std::cout << "Different 'memoryBusWidth': " << prop1.memoryBusWidth << " vs " << prop2.memoryBusWidth << std::endl; + if (prop1.l2CacheSize != prop2.l2CacheSize) + std::cout << "Different 'l2CacheSize': " << prop1.l2CacheSize << " vs " << prop2.l2CacheSize << std::endl; + if (prop1.maxThreadsPerMultiProcessor != prop2.maxThreadsPerMultiProcessor) + std::cout << "Different 'maxThreadsPerMultiProcessor': " << prop1.maxThreadsPerMultiProcessor << " vs " << prop2.maxThreadsPerMultiProcessor << std::endl; + if (prop1.streamPrioritiesSupported != prop2.streamPrioritiesSupported) + std::cout << "Different 'streamPrioritiesSupported': " << prop1.streamPrioritiesSupported << " vs " << prop2.streamPrioritiesSupported << std::endl; + if (prop1.globalL1CacheSupported != prop2.globalL1CacheSupported) + std::cout << "Different 'globalL1CacheSupported': " << prop1.globalL1CacheSupported << " vs " << prop2.globalL1CacheSupported << std::endl; + if (prop1.localL1CacheSupported != prop2.localL1CacheSupported) + std::cout << "Different 'localL1CacheSupported': " << prop1.localL1CacheSupported << " vs " << prop2.localL1CacheSupported << std::endl; + if (prop1.sharedMemPerMultiprocessor != prop2.sharedMemPerMultiprocessor) + std::cout << "Different 'sharedMemPerMultiprocessor': " << prop1.sharedMemPerMultiprocessor << " vs " << prop2.sharedMemPerMultiprocessor << std::endl; + if (prop1.reservedSharedMemPerBlock != prop2.reservedSharedMemPerBlock) + std::cout << "Different 'reservedSharedMemPerBlock': " << prop1.reservedSharedMemPerBlock << " vs " << prop2.reservedSharedMemPerBlock << std::endl; + if (prop1.hostNativeAtomicSupported != prop2.hostNativeAtomicSupported) + std::cout << "Different 'hostNativeAtomicSupported': " << prop1.hostNativeAtomicSupported << " vs " << prop2.hostNativeAtomicSupported << std::endl; + if (prop1.singleToDoublePrecisionPerfRatio != prop2.singleToDoublePrecisionPerfRatio) + std::cout << "Different 'singleToDoublePrecisionPerfRatio': " << prop1.singleToDoublePrecisionPerfRatio << " vs " << prop2.singleToDoublePrecisionPerfRatio << std::endl; + if (prop1.pageableMemoryAccess != prop2.pageableMemoryAccess) + std::cout << "Different 'pageableMemoryAccess': " << prop1.pageableMemoryAccess << " vs " << prop2.pageableMemoryAccess << std::endl; + if (prop1.pageableMemoryAccessUsesHostPageTables != prop2.pageableMemoryAccessUsesHostPageTables) + std::cout << "Different 'pageableMemoryAccessUsesHostPageTables': " << prop1.pageableMemoryAccessUsesHostPageTables << " vs " << prop2.pageableMemoryAccessUsesHostPageTables << std::endl; + if (prop1.directManagedMemAccessFromHost != prop2.directManagedMemAccessFromHost) + std::cout << "Different 'directManagedMemAccessFromHost': " << prop1.directManagedMemAccessFromHost << " vs " << prop2.directManagedMemAccessFromHost << std::endl; +} + +int main(int argc, char* argv[]) { + int gpuid = 0; + cudaGetDevice(&gpuid); + std::cout << "gpuid: " << gpuid << std::endl; + + int cnt = 0; + cudaGetDeviceCount(&cnt); + std::cout<<"cnt: "<name: %s\n", deviceProp.name); + printf("prop->totalGlobalMem: %lu\n", deviceProp.totalGlobalMem); + //printf("prop->sharedMemPerBlock: %lu\n", prop->sharedMemPerBlock); + //printf("prop->regsPerBlock: %d\n", prop->regsPerBlock); + printf("prop->warpSize: %d\n", deviceProp.warpSize); + //printf("prop->memPitch: %lu\n", prop->memPitch); + printf("prop->maxThreadsPerBlock: %d\n", deviceProp.maxThreadsPerBlock); + printf("prop->maxThreadsPerMultiProcessor: %d\n", deviceProp.maxThreadsPerMultiProcessor); + + + // // 打开文件并读取内容 + // std::ifstream inFile("monitor-2.bin", std::ios::in | std::ios::binary); + // if (!inFile) { + // std::cerr << "Failed to open file for reading." << std::endl; + // return 1; + // } + // else { + // std::cout << "File opened: monitor-2.bin" << std::endl; + // } + + // cudaDeviceProp fileDeviceProp; + // inFile.read(reinterpret_cast(&fileDeviceProp), sizeof(cudaDeviceProp)); + + // // 关闭文件 + // inFile.close(); + + // std::cout << "=====================" << std::endl; + // // 比较从文件读取的cudaDeviceProp和从CUDA API获取的cudaDeviceProp + // compareDeviceProps(deviceProp, fileDeviceProp); + // std::cout << "=====================" << std::endl; + + + int val; + cudaDeviceGetAttribute(&val, cudaDevAttrMaxThreadsPerBlock, devID); + std::cout<<"maxThreadsPerBlock: "< +#include +#include +#include + +int main(int argc, char* argv[]) { + // float time_elapsed = 0; + cudaEvent_t start,end; + cudaEventCreate(&start); + cudaEventCreateWithFlags(&end,0); + printf("start = %p, end = %p\n",start,end); + + cudaEventRecord(start,0); + int *dev_A; + cudaMalloc(&dev_A,8); + cudaEventRecord(end,0); + + cudaEventQuery(start); + cudaEventQuery(end); + + cudaEventDestroy(start); + cudaEventDestroy(end); + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testGraph.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testGraph.cu new file mode 100644 index 0000000..fe03d33 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testGraph.cu @@ -0,0 +1,79 @@ +#include +#include + +#define N 1024 * 1024 // 数据大小 +#define THREADS_PER_BLOCK 256 + +__global__ void addKernel(float* data, int n) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx < n) { + data[idx] = data[idx] + 1; + } +} + +// 检查 CUDA 错误 +void checkCudaError(cudaError_t err, const char* msg) { + if (err != cudaSuccess) { + std::cerr << msg << ": " << cudaGetErrorString(err) << std::endl; + exit(EXIT_FAILURE); + } +} + +int main() { + // 1. 初始化数据 + float* h_data = new float[N]; + for (int i = 0; i < N; ++i) { + h_data[i] = static_cast(i); + } + + float* d_data; + checkCudaError(cudaMalloc(&d_data, N * sizeof(float)), "Failed to allocate device memory"); + checkCudaError(cudaMemcpy(d_data, h_data, N * sizeof(float), cudaMemcpyHostToDevice), "Failed to copy data to device"); + + // 2. 创建 CUDA Stream + cudaStream_t stream; + checkCudaError(cudaStreamCreate(&stream), "Failed to create CUDA stream"); + + // 3. CUDA Graph 相关变量 + cudaGraph_t graph; + cudaGraphExec_t graphExec; + + // 4. 开始捕获 CUDA Graph + checkCudaError(cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal), "Failed to begin CUDA Graph capture"); + + // 计算内核配置 + int blocks = (N + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + addKernel<<>>(d_data, N); + + // 停止捕获并生成 CUDA Graph + checkCudaError(cudaStreamEndCapture(stream, &graph), "Failed to end CUDA Graph capture"); + + // 5. 实例化 CUDA Graph + checkCudaError(cudaGraphInstantiate(&graphExec, graph, nullptr, nullptr, 0), "Failed to instantiate CUDA Graph"); + + // 6. 重复执行 CUDA Graph + const int iterations = 5; + for (int i = 0; i < iterations; ++i) { + checkCudaError(cudaGraphLaunch(graphExec, stream), "Failed to launch CUDA Graph"); + checkCudaError(cudaStreamSynchronize(stream), "Failed to synchronize stream"); + } + + cudaDeviceSynchronize(); + + // 7. 将结果从设备复制回主机 + checkCudaError(cudaMemcpy(h_data, d_data, N * sizeof(float), cudaMemcpyDeviceToHost), "Failed to copy data back to host"); + + // 8. 检查结果 + for (int i = 0; i < 5; ++i) { + std::cout << "h_data[" << i << "] = " << h_data[i] << std::endl; + } + + // 9. 释放资源 + cudaGraphExecDestroy(graphExec); + cudaGraphDestroy(graph); + cudaStreamDestroy(stream); + cudaFree(d_data); + delete[] h_data; + + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testHighThread.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testHighThread.cu new file mode 100644 index 0000000..f787014 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testHighThread.cu @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define checkCUDA(expression) \ +{ \ + cudaError_t error = (expression); \ + if (error != cudaSuccess) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudaGetErrorString(error) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +thread_local int t_tid = 0; + +inline int tid() +{ + if (t_tid == 0) + { + t_tid = static_cast(::syscall(SYS_gettid)); + } + return t_tid; +} + +// Function to be executed by each thread +void cudaTask(int priority) { + int device; + + // if (priority == 0) { + // sleep(5); + // } + + + // Get current device + cudaGetDevice(&device); + // printf("Thread %d, Current CUDA device: %d\n", tid(), device); + + // Set device to 0 + checkCUDA(cudaSetDevice(0)); + + // Allocate memory on device + float* dev_A; + size_t size = 256 * 1024 * 1024; // 256 MB + checkCUDA(cudaMalloc((void**)&dev_A, size)); + // printf("Thread %d, Allocated %.2f MB on device %d (%p)\n", tid(), size / (1024.0 * 1024.0), device, dev_A); + + // Example host data initialization + float* host_A = new float[size / sizeof(float)]; + for (size_t i = 0; i < size / sizeof(float); i++) { + host_A[i] = float(i); + } + float* host_B = new float[8]; + + // Asynchronous copy of data from host to device + for (size_t i = 0; i < 100; i++) { + checkCUDA(cudaMemcpyAsync(dev_A, host_A, size, cudaMemcpyHostToDevice)); + checkCUDA(cudaMemcpyAsync(host_B, dev_A, 8 * sizeof(float), cudaMemcpyDeviceToHost)); + } + // printf("Thread %d, Copied data from host to device and back\n", tid()); + // for (size_t i = 0; i < 8; i++) { + // printf("%.2f%c", host_B[i], (i < 7) ? ' ' : '\n'); + // } + checkCUDA(cudaDeviceSynchronize()); + + // Clean up + checkCUDA(cudaFree(dev_A)); + + // Change to device 1 + checkCUDA(cudaSetDevice(1)); + + + delete[] host_A; + delete[] host_B; +} + +int main() { + // Initialize CUDA device + cudaSetDevice(1); + + const int numThreads = 16; + std::vector threads; + + struct timeval timestart; + struct timeval timeend; + gettimeofday(×tart, NULL); + + // Create two threads using STL + for (int i = 0; i < numThreads; ++i) { + threads.emplace_back(cudaTask, i); + } + + // Wait for threads to finish + for (std::thread& t : threads) { + t.join(); + } + + // cudaTask(1); + + gettimeofday(&timeend, NULL); + long diff = 1000000 * (timeend.tv_sec - timestart.tv_sec) + + timeend.tv_usec - timestart.tv_usec; + double msTotalTime = 1.0f * diff / 1000.0; + printf("%lf\n", msTotalTime); + + // std::cout << "All threads have finished." << std::endl; + + return 0; +} + diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testKernelLaunch.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testKernelLaunch.cu new file mode 100644 index 0000000..e906606 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testKernelLaunch.cu @@ -0,0 +1,87 @@ +#include +#include +#include +#include +//#define N 1024 +//#define N 512 + + +__global__ void add(const int *A, const int *B, int *C, size_t n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + C[i] = A[i] + B[i]; +} + +__global__ void sub(const int *A, const int *B, int *C, size_t n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + C[i] = A[i] + B[i]; +} + +int main(int argc, char* argv[]) { + // cudaDeviceSynchronize(); + cudaSetDevice(0); + int testSize = 4096; + const char* logDir; + if (argc == 3) { + testSize = atoi(argv[2]); + logDir = argv[1]; + } + if (argc != 3 && argc != 1){ + printf("Usage: %s [path to the log file] [testSize]\n", argv[0]); + return 0; + } + int* A_arr = (int*)malloc(testSize); + int* B_arr = (int*)malloc(testSize); + int* C_arr = (int*)malloc(testSize); + int num = testSize / sizeof(int); + struct timeval timestart; + struct timeval timeend; + gettimeofday(×tart, NULL); + for (size_t i = 0; i < num; i++) + A_arr[i] = i, B_arr[i] = 1, C_arr[i] = 0; + + int *dev_A, *dev_B, *dev_C; + assert(cudaSuccess == cudaMalloc(&dev_A, testSize)); + assert(cudaSuccess == cudaMalloc(&dev_B, testSize)); + assert(cudaSuccess == cudaMalloc(&dev_C, testSize)); + assert(cudaSuccess == + cudaMemcpy(dev_A, A_arr, testSize, cudaMemcpyHostToDevice)); + assert(cudaSuccess == + cudaMemcpy(dev_B, B_arr, testSize, cudaMemcpyHostToDevice)); + assert(cudaSuccess == + cudaMemcpy(dev_C, C_arr, testSize, cudaMemcpyHostToDevice)); + std::cout << "address of dev_A = " << dev_A << ", dev_B = " << dev_B + << ", dev_C = " << dev_C << std::endl; + + size_t blockDim_x = 512; + //size_t blockDim_x = 256; + size_t nr_block = (num + blockDim_x - 1) / blockDim_x; + std::cout << "nr_block = " << nr_block << ", blockDim_x = " << blockDim_x + << std::endl; + add<<>>(dev_A, dev_B, dev_C, num); + std::cout << "before cudaMemcpy, address of dev_C = " << dev_C << std::endl; + cudaDeviceSynchronize(); + assert(cudaSuccess == + cudaMemcpy(C_arr, dev_C, testSize, cudaMemcpyDeviceToHost)); + + cudaFree(dev_A); + cudaFree(dev_B); + cudaFree(dev_C); + + std::cout << "C = {"; + for (size_t i = num - 10; i < num; i++) + std::cout << C_arr[i] << ", "; + std::cout << "}" << std::endl; + gettimeofday(&timeend, NULL); + long diff = 1000000 * (timeend.tv_sec - timestart.tv_sec) + + timeend.tv_usec - timestart.tv_usec; + double msTotalTime = 1.0f * diff / 1000.0; + printf("Total time = %lf ms\n", msTotalTime); + if (argc == 3) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << msTotalTime << std::endl; + outFile.close(); + } + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testKernelwithStruct.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testKernelwithStruct.cu new file mode 100644 index 0000000..7667af8 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testKernelwithStruct.cu @@ -0,0 +1,82 @@ +#include +#include +#include +#include + +typedef struct { + float a; + float b; + float *z; // pointer to device memory + float *w; // another pointer to device memory +} NestedStruct; + +typedef struct { + int x; + float y; + NestedStruct nested; + float *host_data; // pointer to host memory +} MyStruct; + +__global__ void myKernel(MyStruct myStruct, int offset) { + int idx = threadIdx.x + blockIdx.x * blockDim.x; + myStruct.nested.z[idx] = (myStruct.nested.z[idx] * myStruct.x * myStruct.y * myStruct.nested.a * myStruct.nested.b) + offset; + myStruct.nested.w[idx] = myStruct.nested.z[idx] + myStruct.nested.w[idx]; +} + +int main(int argc, char* argv[]) { + // 定义并初始化结构体 + MyStruct h_myStruct; + h_myStruct.x = 2; + h_myStruct.y = 3.5; + h_myStruct.nested.a = 1.1; // 初始化嵌套结构体成员 + h_myStruct.nested.b = 2.2; + + // 在设备上分配内存并初始化 + float h_z[10] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + float h_w[10] = {10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0}; + float *d_z, *d_w; + cudaMalloc((void**)&d_z, 10 * sizeof(float)); + cudaMalloc((void**)&d_w, 10 * sizeof(float)); + cudaMemcpy(d_z, h_z, 10 * sizeof(float), cudaMemcpyHostToDevice); + cudaMemcpy(d_w, h_w, 10 * sizeof(float), cudaMemcpyHostToDevice); + + // 将设备指针赋值给嵌套结构体 + h_myStruct.nested.z = d_z; + h_myStruct.nested.w = d_w; + + // 在主机上分配内存并初始化 + float h_host_data[10] = {0.0}; + h_myStruct.host_data = h_host_data; + + // 定义offset + int offset = 5; + + // 将结构体作为参数传递给 Kernel 函数，并传递 offset + myKernel<<<1, 10>>>(h_myStruct, offset); + + // 同步并检查错误 + cudaDeviceSynchronize(); + + // 将结果从设备内存复制回主机内存 + cudaMemcpy(h_z, d_z, 10 * sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(h_w, d_w, 10 * sizeof(float), cudaMemcpyDeviceToHost); + + // 打印结果 + printf("Results in z:\n"); + for (int i = 0; i < 10; ++i) { + printf("%f ", h_z[i]); + } + printf("\n"); + + printf("Results in w:\n"); + for (int i = 0; i < 10; ++i) { + printf("%f ", h_w[i]); + } + printf("\n"); + + // 释放设备内存 + cudaFree(d_z); + cudaFree(d_w); + + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testMemory.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testMemory.cu new file mode 100644 index 0000000..9a9fc1b --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testMemory.cu @@ -0,0 +1,82 @@ +/** +* freeMem: 16603348992, totalMem: 16928342016 +* B_arr addr: 0x7f06c7ffe010, data: {16843009,16843009,16843009,16843009,16843009,16843009,16843009,16843009,16843009,16843009} +* C_arr addr: 0x7f06a7ffd010, data: {134217718,134217719,134217720,134217721,134217722,134217723,134217724,134217725,134217726,134217727} +* Total time = 370.274000 ms +*/ +#include +#include +#include +#include +#include +//#define N 4096 + + +int main(int argc, char* argv[]) { + cudaDeviceSynchronize(); + int testSize = 512 * 1024 * 1024; // 4 MB 测试 RNDV + const char* logDir; + if (argc == 3) { + testSize = atoi(argv[2]); + logDir = argv[1]; + } + if (argc != 3 && argc != 1){ + printf("Usage: %s [testSize] [path to the log file]\n", argv[0]); + return 0; + } + size_t freeMem, totalMem; + cudaMemGetInfo(&freeMem, &totalMem); + printf("freeMem: %lu, totalMem: %lu\n", freeMem, totalMem); + + int* A_arr = (int*)malloc(testSize); + int* B_arr = (int*)malloc(testSize); + int* C_arr = (int*)malloc(testSize); + int num = testSize / sizeof(int); + for (size_t i = 0; i < num; i++) + A_arr[i] = i, B_arr[i] = 0, C_arr[i] = 0; + + struct timeval timestart; + struct timeval timeend; + gettimeofday(×tart, NULL); + + int *dev_A, *dev_B; + + assert(cudaSuccess == cudaMalloc(&dev_A, testSize)); + assert(cudaSuccess == cudaMalloc(&dev_B, testSize)); + assert(cudaSuccess == + cudaMemcpy(dev_A, A_arr, testSize, cudaMemcpyHostToDevice)); + assert(cudaSuccess == + cudaMemset(dev_B, 1, testSize)); + // assert(cudaSuccess == + // cudaMemcpy(dev_B, B_arr, testSize, cudaMemcpyHostToDevice)); + // cudaDeviceSynchronize(); + assert(cudaSuccess == + cudaMemcpy(B_arr, dev_B, testSize, cudaMemcpyDeviceToHost)); + assert(cudaSuccess == + cudaMemcpy(C_arr, dev_A, testSize, cudaMemcpyDeviceToHost)); + //cudaDeviceSynchronize(); + + printf("B_arr addr: %p, data: {", B_arr); + + + for (size_t i = max(num - 10, 0); i < num; i++) // 16843009 = 0000 0001 0000 0001 0000 0001 0000 0001 + printf("%d%s", B_arr[i], (i == num - 1) ? "}\n" : ","); + + printf("C_arr addr: %p, data: {", C_arr); + for (size_t i = max(num - 10, 0); i < num; i++) + printf("%d%s", C_arr[i], (i == num - 1) ? "}\n" : ","); + + cudaFree(dev_A); + cudaFree(dev_B); + gettimeofday(&timeend, NULL); + long diff = 1000000 * (timeend.tv_sec - timestart.tv_sec) + + timeend.tv_usec - timestart.tv_usec; + double msTotalTime = 1.0f * diff / 1000.0; + printf("Total time = %lf ms\n", msTotalTime); + if (argc == 3) { + std::ofstream outFile(logDir, std::ios::out | std::ios::app); + outFile << msTotalTime << std::endl; + outFile.close(); + } + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testMemory2.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testMemory2.cu new file mode 100644 index 0000000..a6b7e12 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testMemory2.cu @@ -0,0 +1,41 @@ +// main.cu +#include +#include +#include +#include +#include +//#define N 4096 + + +int main(int argc, char* argv[]) { + int nelem = 1048576; + size_t bytes = nelem * sizeof(int); + + + size_t freeMem, totalMem; + cudaMemGetInfo(&freeMem, &totalMem); + printf("freeMem: %lu, totalMem: %lu\n", freeMem, totalMem); + + int *A_arr, *B_arr, *C_arr; + // testcodes: https://github.com/NVIDIA/cuda-samples/blob/e8568c417356f7e66bb9b7130d6be7e55324a519/Samples/2_Concepts_and_Techniques/inlinePTX/inlinePTX.cu#L82 + cudaMallocHost((void**)&A_arr, bytes); + //cudaMallocHost((void**)&B_arr, bytes); + cudaHostAlloc((void**)&B_arr, bytes, cudaHostAllocDefault); + // A_arr = (int*)malloc(bytes); + // B_arr = (int*)malloc(bytes); + C_arr = (int*)malloc(bytes); + + for (int i = 0; i < nelem; i++) { + A_arr[i] = i; + B_arr[i] = 2; + C_arr[i] = 0; + } + int idx = nelem - 2; + printf("A_arr addr = %p, A_arr[%d] = %d\n", A_arr, idx, A_arr[idx]); + printf("B_arr addr = %p, B_arr[%d] = %d\n", B_arr, idx, B_arr[idx]); + + + + + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclClient.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclClient.cu new file mode 100644 index 0000000..ae12cd0 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclClient.cu @@ -0,0 +1,198 @@ +#include "./include/nccl_util.h" + +int main(int argc, char *argv[]) { + int version; + NCCLCHECK(ncclGetVersion(&version)); + printf("NCCL version: %d\n", version); + + // Initialize client socket + const char* server_ip = (argc > 1) ? argv[1] : SERVER_IP; + int sock = 0; + struct sockaddr_in serv_addr; + + if ((sock = socket(AF_INET, SOCK_STREAM, 0)) < 0) { + printf("\n Socket creation error \n"); + return -1; + } + + serv_addr.sin_family = AF_INET; + serv_addr.sin_port = htons(SERVER_PORT); + + if (inet_pton(AF_INET, server_ip, &serv_addr.sin_addr) <= 0) { + printf("\nInvalid address/ Address not supported \n"); + return -1; + } + + if (connect(sock, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + printf("\nConnection Failed \n"); + return -1; + } + + // Receiving NCCL unique ID from server + ncclUniqueId id; + int nRanks; + int baseRank; + ssize_t bytesRead = read(sock, &id, sizeof(id)); + if (bytesRead < 0) { + printf("\nError reading NCCL unique ID\n"); + return -1; + } + bytesRead = read(sock, &nRanks, sizeof(nRanks)); + if (bytesRead < 0) { + printf("\nError reading nRanks\n"); + return -1; + } + bytesRead = read(sock, &baseRank, sizeof(baseRank)); + if (bytesRead < 0) { + printf("\nError reading baseRank\n"); + return -1; + } + close(sock); + printf("Received NCCL unique ID, nRanks=%d, baseRank=%d\n", nRanks, baseRank); + + // Initialize CUDA resources + float **sendbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **recvbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **allgatherbuff = (float **)malloc(BUFFER_SIZE * nRanks * sizeof(float *)); + float **reducescatterbuff = (float **)malloc(BUFFER_SIZE / nRanks * sizeof(float *)); + float **reducebuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **broadcastbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **sendrecvbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + cudaStream_t *s = (cudaStream_t *)malloc(sizeof(cudaStream_t) * GPUS_NUM); + float* tmpbuff = (float*)malloc(BUFFER_SIZE * sizeof(float)); + for (int i = 0; i < BUFFER_SIZE; i++) tmpbuff[i] = 2; + + for (int devID = 0; devID < GPUS_NUM; ++devID) { + CUDACHECK(cudaSetDevice(devID)); + CUDACHECK(cudaStreamCreate(s + devID)); + CUDACHECK(cudaMalloc(sendbuff + devID, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(recvbuff + devID, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(allgatherbuff + devID, BUFFER_SIZE * nRanks * sizeof(float))); + CUDACHECK(cudaMalloc(reducescatterbuff + devID, BUFFER_SIZE / nRanks * sizeof(float))); + CUDACHECK(cudaMalloc(reducebuff + devID, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(broadcastbuff + devID, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(sendrecvbuff + devID, BUFFER_SIZE * sizeof(float))); + + CUDACHECK(cudaMemcpy(sendbuff[devID], tmpbuff, BUFFER_SIZE * sizeof(float), cudaMemcpyHostToDevice)); + + CUDACHECK(cudaMemset(recvbuff[devID], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(allgatherbuff[devID], 0, BUFFER_SIZE * nRanks * sizeof(float))); + CUDACHECK(cudaMemset(reducescatterbuff[devID], 0, BUFFER_SIZE / nRanks * sizeof(float))); + CUDACHECK(cudaMemset(reducebuff[devID], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(broadcastbuff[devID], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(sendrecvbuff[devID], 0, BUFFER_SIZE * sizeof(float))); + } + printf("Init CUDA resources\n"); + printf("Before operations:\n"); + PrintBuffer((void**)sendbuff); + + // Initializing NCCL + ncclComm_t comms[GPUS_NUM]; + ncclConfig_t config = NCCL_CONFIG_INITIALIZER; + config.netName = "Socket"; + config.blocking = 1; + NCCLCHECK(ncclGroupStart()); + for (int devID = 0; devID < GPUS_NUM; devID++) { + CUDACHECK(cudaSetDevice(devID)); +#ifdef USE_CONFIG + NCCLCHECK(ncclCommInitRankConfig(comms + devID, nRanks, id, baseRank + devID, &config)); +#else + NCCLCHECK(ncclCommInitRank(comms + devID, nRanks, id, baseRank + devID)); +#endif + + } + NCCLCHECK(ncclGroupEnd()); + + for (int devID = 0; devID < GPUS_NUM; devID++) { + int count, device, userRank; + NCCLCHECK(ncclCommCount(comms[devID], &count)); + NCCLCHECK(ncclCommCuDevice(comms[devID], &device)); + NCCLCHECK(ncclCommUserRank(comms[devID], &userRank)); + printf("comm: %p, CUDADevice: %d, userRank: %d, nRanks: %d\n", comms[devID], device, userRank, count); + } + + // NCCL communication + NCCLCHECK(ncclGroupStart()); + for (int devID = 0; devID < GPUS_NUM; devID++) { + // AllReduce + NCCLCHECK(ncclAllReduce((const void *)sendbuff[devID], (void *)recvbuff[devID], + BUFFER_SIZE, ncclFloat, ncclSum, comms[devID], s[devID])); + + // AllGather + NCCLCHECK(ncclAllGather((const void *)sendbuff[devID], (void *)allgatherbuff[devID], + BUFFER_SIZE, ncclFloat, comms[devID], s[devID])); + + // ReduceScatter + NCCLCHECK(ncclReduceScatter((const void *)sendbuff[devID], (void *)reducescatterbuff[devID], + BUFFER_SIZE / nRanks, ncclFloat, ncclSum, comms[devID], s[devID])); + + // Reduce + NCCLCHECK(ncclReduce((const void *)sendbuff[devID], (void *)reducebuff[devID], + BUFFER_SIZE, ncclFloat, ncclSum, 0, comms[devID], s[devID])); + + // Broadcast + NCCLCHECK(ncclBroadcast((const void *)sendbuff[0], (void *)broadcastbuff[devID], + BUFFER_SIZE, ncclFloat, 0, comms[devID], s[devID])); + } + NCCLCHECK(ncclGroupEnd()); + + // Send and Recv (must be grouped together) + NCCLCHECK(ncclGroupStart()); + for (int devID = 0; devID < GPUS_NUM; devID++) { + if (devID == 0) { + // dev #0 (client, rank2) sends to dev rank0 (server) + NCCLCHECK(ncclSend((const void *)sendbuff[devID], BUFFER_SIZE, ncclFloat, 0, comms[devID], s[devID])); + } + else if (devID == 1) { + // dev #1 (client, rank3) receives from dev rank1 (server) + NCCLCHECK(ncclRecv((void *)sendrecvbuff[devID], BUFFER_SIZE, ncclFloat, 1, comms[devID], s[devID])); + } + } + NCCLCHECK(ncclGroupEnd()); + + // Synchronizing on CUDA stream + for (int i = 0; i < GPUS_NUM; i++) { + CUDACHECK(cudaSetDevice(i)); + CUDACHECK(cudaStreamSynchronize(s[i])); + ncclResult_t ncclAsyncErr; + NCCLCHECK(ncclCommGetAsyncError(comms[i], &ncclAsyncErr)); + if (ncclAsyncErr != ncclSuccess) { + // An asynchronous error happened. Stop the operation and destroy + // the communicator + NCCLCHECK(ncclCommAbort(comms[i])); + } + } + printf("After operations:\n"); + printf("AllReduce result:\n"); + PrintBuffer((void**)recvbuff); + printf("AllGather result:\n"); + PrintBuffer((void**)allgatherbuff, BUFFER_SIZE * nRanks); + printf("ReduceScatter result:\n"); + PrintBuffer((void**)reducescatterbuff, BUFFER_SIZE / nRanks); + printf("Reduce result:\n"); + PrintBuffer((void**)reducebuff); + printf("Broadcast result:\n"); + PrintBuffer((void**)broadcastbuff); + printf("SendRecv result:\n"); + PrintBuffer((void**)sendrecvbuff); + + // Freeing device memory + for (int i = 0; i < GPUS_NUM; i++) { + CUDACHECK(cudaSetDevice(i)); + CUDACHECK(cudaFree(sendbuff[i])); + CUDACHECK(cudaFree(recvbuff[i])); + CUDACHECK(cudaFree(allgatherbuff[i])); + CUDACHECK(cudaFree(reducescatterbuff[i])); + CUDACHECK(cudaFree(reducebuff[i])); + CUDACHECK(cudaFree(broadcastbuff[i])); + CUDACHECK(cudaFree(sendrecvbuff[i])); + } + + // Finalizing NCCL + for (int devID = 0; devID < GPUS_NUM; devID++) { + NCCLCHECK(ncclCommDestroy(comms[devID])); + } + + printf("Client completed successfully\n"); + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclLocal.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclLocal.cu new file mode 100644 index 0000000..9c9a5c9 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclLocal.cu @@ -0,0 +1,147 @@ +#include "./include/nccl_util.h" + +int main(int argc, char *argv[]) { + + // managing 4 devices + int nDev = GPUS_NUM; + int devs[GPUS_NUM]; + for (int i = 0; i < nDev; ++i) devs[i] = i; + + ncclComm_t comms[GPUS_NUM]; + + // allocating and initializing device buffers + float **sendbuff = (float **)malloc(nDev * sizeof(float *)); + float **recvbuff = (float **)malloc(nDev * sizeof(float *)); + float **recvbuff2 = (float **)malloc(nDev * sizeof(float *)); + void *sendRegHandle[nDev]; + void *recvRegHandle[nDev]; + cudaStream_t *s = (cudaStream_t *)malloc(sizeof(cudaStream_t) * nDev); + float* tmpbuff = (float*)malloc(BUFFER_SIZE * sizeof(float)); + for (int i = 0; i < BUFFER_SIZE; i++) tmpbuff[i] = 1; + + for (int i = 0; i < nDev; ++i) { + CUDACHECK(cudaSetDevice(i)); + + NCCLCHECK(ncclMemAlloc((void **)sendbuff + i, BUFFER_SIZE * sizeof(float))); + NCCLCHECK(ncclMemAlloc((void **)recvbuff + i, BUFFER_SIZE * sizeof(float))); + + // CUDACHECK(cudaMalloc((void**)sendbuff + i, size * sizeof(float))); + NCCLCHECK(ncclMemAlloc((void **)recvbuff2 + i, BUFFER_SIZE * sizeof(float))); + + CUDACHECK(cudaMemcpy(sendbuff[i], tmpbuff, BUFFER_SIZE * sizeof(float), cudaMemcpyHostToDevice)); + + CUDACHECK(cudaMemset(recvbuff[i], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(recvbuff2[i], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaStreamCreate(s + i)); + } + + // initializing NCCL and preparing custom operation + NCCLCHECK(ncclCommInitAll(comms, nDev, devs)); + float scalar_1 = 2.0f, scalar_2 = 3.0f; + float *d_scalar; + // CUDACHECK(cudaMalloc(&d_scalar, sizeof(float))); + NCCLCHECK(ncclMemAlloc((void **)&d_scalar, sizeof(float))); + CUDACHECK( + cudaMemcpy(d_scalar, &scalar_2, sizeof(float), cudaMemcpyHostToDevice)); + ncclRedOp_t customOp_1[nDev], customOp_2[nDev]; + for (int i = 0; i < nDev; ++i) { + NCCLCHECK(ncclCommRegister(comms[i], sendbuff[i], BUFFER_SIZE * sizeof(float), + sendRegHandle + i)); + NCCLCHECK(ncclCommRegister(comms[i], recvbuff[i], BUFFER_SIZE * sizeof(float), + recvRegHandle + i)); + NCCLCHECK(ncclRedOpCreatePreMulSum(customOp_1 + i, &scalar_1, ncclFloat, + ncclScalarHostImmediate, comms[i])); + NCCLCHECK(ncclRedOpCreatePreMulSum(customOp_2 + i, d_scalar, ncclFloat, + ncclScalarDevice, comms[i])); + printf("CustomOp_1: %u, CustomOp_2: %u\n", customOp_1[i], customOp_2[i]); + } + + printf("Before AllGather:\n"); + PrintBuffer((void**)sendbuff); + + // calling NCCL communication API. Group API is required when using + // multiple devices per thread + NCCLCHECK(ncclGroupStart()); + for (int i = 0; i < nDev; ++i) { + NCCLCHECK(ncclAllReduce((const void *)sendbuff[i], (void *)recvbuff[i], + BUFFER_SIZE, ncclFloat, customOp_1[i], comms[i], s[i])); + } + NCCLCHECK(ncclGroupEnd()); + + // synchronizing on CUDA streams to wait for completion of NCCL operation + for (int i = 0; i < nDev; ++i) { + CUDACHECK(cudaSetDevice(i)); + CUDACHECK(cudaStreamSynchronize(s[i])); + } + +// NCCLCHECK(ncclGroupStart()); +// for (int i = 0; i < nDev; ++i) { +// NCCLCHECK(ncclAllReduce((const void *)sendbuff[i], (void *)recvbuff2[i], +// size, ncclFloat, customOp_2[i], comms[i], s[i])); +// } +// NCCLCHECK(ncclGroupEnd()); + +// // synchronizing on CUDA streams to wait for completion of NCCL operation +// for (int i = 0; i < nDev; ++i) { +// CUDACHECK(cudaSetDevice(i)); +// CUDACHECK(cudaStreamSynchronize(s[i])); +// } + + printf("AllGather result:\n"); + PrintBuffer((void**)recvbuff); + + // free device buffers + for (int i = 0; i < nDev; ++i) { + CUDACHECK(cudaSetDevice(i)); + NCCLCHECK(ncclCommDeregister(comms[i], sendRegHandle[i])); + NCCLCHECK(ncclCommDeregister(comms[i], recvRegHandle[i])); + NCCLCHECK(ncclMemFree(sendbuff[i])); + NCCLCHECK(ncclMemFree(recvbuff[i])); + NCCLCHECK(ncclMemFree(recvbuff2[i])); + // CUDACHECK(cudaFree(sendbuff[i])); + // CUDACHECK(cudaFree(recvbuff2[i])); + } + + + // finalizing NCCL + for (int i = 0; i < nDev; ++i) { + ncclRedOpDestroy(customOp_1[i], comms[i]); + ncclRedOpDestroy(customOp_2[i], comms[i]); + } + + for (int i =0; i < nDev; ++i) { + // Finalize the NCCL communicator + NCCLCHECK(ncclCommFinalize(comms[i])); + + // Check for communicator state + ncclResult_t asyncErr; + do { + NCCLCHECK(ncclCommGetAsyncError(comms[i], &asyncErr)); + } while (asyncErr == ncclInProgress); + + if (asyncErr == ncclSuccess) { + printf("Communicator successfully finalized.\n"); + NCCLCHECK(ncclCommDestroy(comms[i])); + } else { + printf("Failed to finalize communicator.\n"); + } + } + + // ncclUniqueId commId; + // NCCLCHECK(ncclGetUniqueId(&commId)); + // ncclConfig_t config = NCCL_CONFIG_INITIALIZER; + // config.netName = "Socket"; + // ncclComm_t newcomm[nDev]; + // NCCLCHECK(ncclGroupStart()); + // for (int i = 0; i < nDev; ++i) { + // NCCLCHECK(ncclCommInitRankConfig(newcomm + i, nDev, commId, i, &config)); + // } + // NCCLCHECK(ncclGroupEnd()); + + // for (int i = 0; i < nDev; ++i) { + // NCCLCHECK(ncclCommDestroy(newcomm[i])); + // } + + printf("Success \n"); + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclServer.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclServer.cu new file mode 100644 index 0000000..d47c7e9 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testNcclServer.cu @@ -0,0 +1,216 @@ +#include "./include/nccl_util.h" + +int main(int argc, char *argv[]) { + int version; + NCCLCHECK(ncclGetVersion(&version)); + printf("NCCL version: %d\n", version); + + const char* server_ip = (argc > 1) ? argv[1] : SERVER_IP; + int nRanks = GPUS_NUM * (CLIENTS_NUM + 1); // each node has same number of GPUs + + // Initialize server socket + int server_fd, new_socket; + struct sockaddr_in address; + int opt = 1; + int addrlen = sizeof(address); + + if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) { + perror("socket failed"); + exit(EXIT_FAILURE); + } + + if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, &opt, + sizeof(opt))) { + perror("setsockopt"); + close(server_fd); + exit(EXIT_FAILURE); + } + + address.sin_family = AF_INET; + address.sin_port = htons(SERVER_PORT); + if (inet_pton(AF_INET, server_ip, &address.sin_addr) <= 0) { + printf("\nInvalid address/ Address not supported \n"); + return -1; + } + + if (bind(server_fd, (struct sockaddr *)&address, sizeof(address)) < 0) { + perror("bind failed"); + close(server_fd); + exit(EXIT_FAILURE); + } + + if (listen(server_fd, 3) < 0) { + perror("listen"); + close(server_fd); + exit(EXIT_FAILURE); + } + printf("Server listening on port %s:%d\n", server_ip, SERVER_PORT); + + // Generating NCCL unique ID + ncclUniqueId id; + ncclGetUniqueId(&id); + + // Accepting connections from clients + for (int i = 0; i < CLIENTS_NUM; ++i) { + int clientBaseRank = GPUS_NUM + i * GPUS_NUM; //start behind server + if ((new_socket = accept(server_fd, (struct sockaddr *)&address, + (socklen_t *)&addrlen)) < 0) { + perror("accept"); + close(server_fd); + exit(EXIT_FAILURE); + } + // Sending NCCL unique ID, nRanks, and its base rank to clients + send(new_socket, &id, sizeof(id), 0); + send(new_socket, &nRanks, sizeof(nRanks), 0); + send(new_socket, &clientBaseRank, sizeof(clientBaseRank), 0); + close(new_socket); + } + printf("NCCL unique ID, nRanks and base rank sent to clients\n"); + + float **sendbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **recvbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **allgatherbuff = (float **)malloc(BUFFER_SIZE * nRanks * sizeof(float *)); + float **reducescatterbuff = (float **)malloc(BUFFER_SIZE / nRanks * sizeof(float *)); + float **reducebuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **broadcastbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + float **sendrecvbuff = (float **)malloc(GPUS_NUM * sizeof(float *)); + cudaStream_t *s = (cudaStream_t *)malloc(sizeof(cudaStream_t) * GPUS_NUM); + // Initialize CUDA resources + float* tmpbuff = (float*)malloc(BUFFER_SIZE * sizeof(float)); + for (int i = 0; i < BUFFER_SIZE; i++) tmpbuff[i] = 1; + + for (int I = 0; I < GPUS_NUM; ++I) { + CUDACHECK(cudaSetDevice(I)); + CUDACHECK(cudaStreamCreate(s + I)); + CUDACHECK(cudaMalloc(sendbuff + I, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(recvbuff + I, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(allgatherbuff + I, BUFFER_SIZE * nRanks * sizeof(float))); + CUDACHECK(cudaMalloc(reducescatterbuff + I, BUFFER_SIZE / nRanks * sizeof(float))); + CUDACHECK(cudaMalloc(reducebuff + I, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(broadcastbuff + I, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMalloc(sendrecvbuff + I, BUFFER_SIZE * sizeof(float))); + + CUDACHECK(cudaMemcpy(sendbuff[I], tmpbuff, BUFFER_SIZE * sizeof(float), cudaMemcpyHostToDevice)); + + CUDACHECK(cudaMemset(recvbuff[I], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(allgatherbuff[I], 0, BUFFER_SIZE * nRanks * sizeof(float))); + CUDACHECK(cudaMemset(reducescatterbuff[I], 0, BUFFER_SIZE / nRanks * sizeof(float))); + CUDACHECK(cudaMemset(reducebuff[I], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(broadcastbuff[I], 0, BUFFER_SIZE * sizeof(float))); + CUDACHECK(cudaMemset(sendrecvbuff[I], 0, BUFFER_SIZE * sizeof(float))); + } + printf("Init CUDA resources\n"); + printf("Before operations:\n"); + PrintBuffer((void**)sendbuff); + + // Initializing NCCL + ncclConfig_t config = NCCL_CONFIG_INITIALIZER; + config.netName = "Socket"; + ncclComm_t comms[GPUS_NUM]; + NCCLCHECK(ncclGroupStart()); + for (int devID = 0; devID < GPUS_NUM; devID++) { + CUDACHECK(cudaSetDevice(devID)); +#ifdef USE_CONFIG + NCCLCHECK(ncclCommInitRankConfig(comms + devID, nRanks, id, devID, &config)); +#else + NCCLCHECK(ncclCommInitRank(comms + devID, nRanks, id, devID)); +#endif + } + NCCLCHECK(ncclGroupEnd()); + + for (int devID = 0; devID < GPUS_NUM; devID++) { + int count, device, userRank; + NCCLCHECK(ncclCommCount(comms[devID], &count)); + NCCLCHECK(ncclCommCuDevice(comms[devID], &device)); + NCCLCHECK(ncclCommUserRank(comms[devID], &userRank)); + printf("comm: %p, CUDADevice: %d, userRank: %d, nRanks: %d\n", comms[devID], device, userRank, count); + } + + // NCCL communication + NCCLCHECK(ncclGroupStart()); + for (int devID = 0; devID < GPUS_NUM; devID++) { + // AllReduce + NCCLCHECK(ncclAllReduce((const void *)sendbuff[devID], (void *)recvbuff[devID], + BUFFER_SIZE, ncclFloat, ncclSum, comms[devID], s[devID])); + + // AllGather + NCCLCHECK(ncclAllGather((const void *)sendbuff[devID], (void *)allgatherbuff[devID], + BUFFER_SIZE, ncclFloat, comms[devID], s[devID])); + + // ReduceScatter + NCCLCHECK(ncclReduceScatter((const void *)sendbuff[devID], (void *)reducescatterbuff[devID], + BUFFER_SIZE / nRanks, ncclFloat, ncclSum, comms[devID], s[devID])); + + // Reduce + NCCLCHECK(ncclReduce((const void *)sendbuff[devID], (void *)reducebuff[devID], + BUFFER_SIZE, ncclFloat, ncclSum, 0, comms[devID], s[devID])); + + // Broadcast + NCCLCHECK(ncclBroadcast((const void *)sendbuff[0], (void *)broadcastbuff[devID], + BUFFER_SIZE, ncclFloat, 0, comms[devID], s[devID])); + + // Send and Recv + + } + NCCLCHECK(ncclGroupEnd()); + + // Send and Recv + NCCLCHECK(ncclGroupStart()); + for (int devID = 0; devID < GPUS_NUM; devID++) { + if (devID == 1) { + // dev #1 (server, rank1) sends to dev rank3 (client) + NCCLCHECK(ncclSend((const void *)sendbuff[devID], BUFFER_SIZE, ncclFloat, 3, comms[devID], s[devID])); + } + else if (devID == 0) { + // dev #0 (server, rank0) receives from dev rank2 (client) + NCCLCHECK(ncclRecv((void *)sendrecvbuff[devID], BUFFER_SIZE, ncclFloat, 2, comms[devID], s[devID])); + } + } + NCCLCHECK(ncclGroupEnd()); + + // Synchronizing on CUDA stream + for (int i = 0; i < GPUS_NUM; i++) { + CUDACHECK(cudaSetDevice(i)); + CUDACHECK(cudaStreamSynchronize(s[i])); + ncclResult_t ncclAsyncErr; + NCCLCHECK(ncclCommGetAsyncError(comms[i], &ncclAsyncErr)); + if (ncclAsyncErr != ncclSuccess) { + // An asynchronous error happened. Stop the operation and destroy + // the communicator + NCCLCHECK(ncclCommAbort(comms[i])); + } + } + printf("After operations:\n"); + printf("AllReduce result:\n"); + PrintBuffer((void**)recvbuff); + printf("AllGather result:\n"); + PrintBuffer((void**)allgatherbuff, BUFFER_SIZE * nRanks); + printf("ReduceScatter result:\n"); + PrintBuffer((void**)reducescatterbuff, BUFFER_SIZE / nRanks); + printf("Reduce result:\n"); + PrintBuffer((void**)reducebuff); + printf("Broadcast result:\n"); + PrintBuffer((void**)broadcastbuff); + printf("SendRecv result:\n"); + PrintBuffer((void**)sendrecvbuff); + + // Freeing device memory + for (int devID = 0; devID < GPUS_NUM; devID++) { + CUDACHECK(cudaSetDevice(devID)); + CUDACHECK(cudaFree(sendbuff[devID])); + CUDACHECK(cudaFree(recvbuff[devID])); + CUDACHECK(cudaFree(allgatherbuff[devID])); + CUDACHECK(cudaFree(reducescatterbuff[devID])); + CUDACHECK(cudaFree(reducebuff[devID])); + CUDACHECK(cudaFree(broadcastbuff[devID])); + } + + // Finalizing NCCL + for (int devID = 0; devID < GPUS_NUM; devID++) { + ncclCommDestroy(comms[devID]); + } + + printf("Server completed successfully\n"); + close(server_fd); + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testOthers.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testOthers.cu new file mode 100644 index 0000000..6e9b6d1 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testOthers.cu @@ -0,0 +1,32 @@ +#include +#include +#include +#include +#include + +#define DATA_SIZE (64ULL << 20ULL) // 64MB + +__global__ void simpleKernel(char *ptr, int sz, char val) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + for (; idx < sz; idx += (gridDim.x * blockDim.x)) { + ptr[idx] = val; + } +} + +int main() { + int blocks = 0; + int threads = 128; + cudaDeviceProp prop; + // cudaStream_t stream; + // char id = 'x'; + + cudaGetDeviceProperties(&prop, 0); + std::cout << "SM count: " << prop.multiProcessorCount << std::endl; + cudaOccupancyMaxActiveBlocksPerMultiprocessor(&blocks, simpleKernel, threads, 0); + std::cout << "Max blocks per SM: " << blocks << std::endl; + + cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(&blocks, simpleKernel, threads, 0, cudaOccupancyDisableCachingOverride); + std::cout << "Max blocks per SM (no caching): " << blocks << std::endl; + + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testStream.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testStream.cu new file mode 100644 index 0000000..e747ec4 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testStream.cu @@ -0,0 +1,68 @@ +#include +#include +#include +#include +#include +#include + + +int main() { + + int leastPriority, greatestPriority; + cudaDeviceGetStreamPriorityRange(&leastPriority, &greatestPriority); + if (leastPriority == greatestPriority) { + printf("This device does not support stream priority.\n"); + } else { + printf("Priority range: [%d, %d]\n", greatestPriority, leastPriority); + } + + // Create a CUDA stream with flags + cudaStream_t stream; + assert(cudaSuccess == cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); + + // Create a CUDA stream with priority + cudaStream_t streamWithPriority; + assert(cudaSuccess == cudaStreamCreateWithPriority(&streamWithPriority, cudaStreamNonBlocking, 1)); + + // Create an event + cudaEvent_t event; + assert(cudaSuccess == cudaEventCreate(&event)); + + printf("stream = %p, streamWithPriority = %p, event = %p\n", stream, streamWithPriority, event); + + // Record an event in the stream + // cudaEventRecord(event, stream); + + // Wait for the event to complete in another stream + assert(cudaSuccess == cudaStreamWaitEvent(streamWithPriority, event, 0)); + + // Synchronize streams + assert(cudaSuccess == cudaStreamSynchronize(stream)); + assert(cudaSuccess == cudaStreamSynchronize(streamWithPriority)); + + // Check if a stream is capturing + enum cudaStreamCaptureStatus capturing; + assert(cudaSuccess == cudaStreamIsCapturing(stream, &capturing)); + if (capturing) { + std::cout << "Stream is capturing." << std::endl; + } else { + std::cout << "Stream is not capturing." << std::endl; + } + + // Get capture information + enum cudaStreamCaptureStatus status; + unsigned long long globalThreadId; + assert(cudaSuccess == cudaStreamGetCaptureInfo(stream, &status, &globalThreadId)); + std::cout << "Capture Status: " << status << std::endl; + std::cout << "Global Thread ID: " << globalThreadId << std::endl; + + // Clean up + assert(cudaSuccess == cudaStreamDestroy(stream)); + assert(cudaSuccess == cudaStreamDestroy(streamWithPriority)); + assert(cudaSuccess == cudaEventDestroy(event)); + + return 0; +} + + +//end \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testT.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testT.cu new file mode 100644 index 0000000..e281dc2 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testT.cu @@ -0,0 +1,88 @@ +#include +#include +#include + +// 错误检查宏 +#define CHECK_CUDA(call) { \ + cudaError_t cudaStatus = call; \ + if (cudaStatus != cudaSuccess) { \ + fprintf(stderr, "CUDA Error: %s\n", cudaGetErrorString(cudaStatus)); \ + exit(-1); \ + } \ +} + +// 核函数 +__global__ void addOne(float *data, int size) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < size) { + data[index] += 1.0f; + } +} + +int main() { + const int size = 9; + float *h_data = new float[size]; // 主机端内存 + float *d_data; // 设备端内存 + const int bytes = size * sizeof(float); + + // 初始化数据 + for (int i = 0; i < size; ++i) { + h_data[i] = 1.11111111111f * (i + 1); + } + + // 在 GPU 上分配内存 + CHECK_CUDA(cudaMalloc((void **)&d_data, bytes)); + + // 异步拷贝数据到 GPU + CHECK_CUDA(cudaMemcpyAsync(d_data, h_data, bytes, cudaMemcpyHostToDevice)); + + // 确定线程块大小和网格大小 + int blockSize = 256; + int gridSize = (size + blockSize - 1) / blockSize; + + struct cudaFuncAttributes funcAttrib; + CHECK_CUDA(cudaFuncGetAttributes(&funcAttrib, addOne)); + std::cout << "Kernel Function Info:" << "\n" + << " Max Threads Per Block: " << funcAttrib.maxThreadsPerBlock << "\n" + << " Shared Memory Per Block: " << funcAttrib.sharedSizeBytes << "\n" + << " Constant Memory Per Block: " << funcAttrib.constSizeBytes << "\n" + << " Local Memory Per Block: " << funcAttrib.localSizeBytes << "\n" + << " Registers Per Block: " << funcAttrib.numRegs << "\n" + << " PTX Version: " << funcAttrib.ptxVersion << "\n" + << " Binary Version: " << funcAttrib.binaryVersion << std::endl; + + // 启动核函数 + addOne<<>>(d_data, size); + CHECK_CUDA(cudaPeekAtLastError()); // 检查核函数是否启动成功 + + // 异步拷贝结果回主机 + CHECK_CUDA(cudaMemcpyAsync(h_data, d_data, bytes, cudaMemcpyDeviceToHost)); + + // 等待 GPU 完成 + CHECK_CUDA(cudaDeviceSynchronize()); + + // 打印结果 + std::cout << "Results:" << std::endl; + for (int i = 0; i < size; ++i) { + std::cout << h_data[i] << std::endl; + } + + // 清理 + delete[] h_data; + cudaFree(d_data); + + // // 假设有一个CUDA操作，这里直接使用一个错误码模拟 + // CUresult result = CUDA_ERROR_INVALID_VALUE; // 示例错误码 + + // // 获取错误描述字符串 + // const char* errorStr; + // CUresult status = cuGetErrorString(result, &errorStr); + + // if (status == CUDA_SUCCESS) { + // std::cout << "CUDA Error: " << errorStr << std::endl; + // } else { + // std::cout << "Failed to get CUDA error string." << std::endl; + // } + + return 0; +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testThread.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testThread.cu new file mode 100644 index 0000000..baa1b19 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testThread.cu @@ -0,0 +1,102 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#define checkCUDA(expression) \ +{ \ + cudaError_t error = (expression); \ + if (error != cudaSuccess) { \ + std::cerr << "Error on line " << __LINE__ << ": " \ + << cudaGetErrorString(error) << std::endl; \ + std::exit(EXIT_FAILURE); \ + } \ +} + +thread_local int t_tid = 0; + +inline int tid() +{ + if (t_tid == 0) + { + t_tid = static_cast(::syscall(SYS_gettid)); + } + return t_tid; +} + +// Function to be executed by each thread +void cudaTask(int priority) { + int device; + + // if (priority == 0) { + // sleep(5); + // } + + + // Get current device + cudaGetDevice(&device); + printf("Thread %d, Current CUDA device: %d\n", tid(), device); + + // Set device to 0 + checkCUDA(cudaSetDevice(0)); + + // Allocate memory on device + float* dev_A; + size_t size = 256 * 1024 * 1024; // 256 MB + checkCUDA(cudaMalloc((void**)&dev_A, size)); + printf("Thread %d, Allocated %.2f MB on device %d (%p)\n", tid(), size / (1024.0 * 1024.0), device, dev_A); + + // Example host data initialization + float* host_A = new float[size / sizeof(float)]; + for (size_t i = 0; i < size / sizeof(float); i++) { + host_A[i] = float(i); + } + float* host_B = new float[8]; + + // Asynchronous copy of data from host to device + checkCUDA(cudaMemcpyAsync(dev_A, host_A, size, cudaMemcpyHostToDevice)); + checkCUDA(cudaMemcpyAsync(host_B, dev_A, 8 * sizeof(float), cudaMemcpyDeviceToHost)); + printf("Thread %d, Copied data from host to device and back\n", tid()); + for (size_t i = 0; i < 8; i++) { + printf("%.2f%c", host_B[i], (i < 7) ? ' ' : '\n'); + } + + // Clean up + checkCUDA(cudaFree(dev_A)); + + // Change to device 1 + checkCUDA(cudaSetDevice(1)); + + + delete[] host_A; + delete[] host_B; +} + +int main() { + // Initialize CUDA device + cudaSetDevice(1); + + const int numThreads = 16; + std::vector threads; + + // Create two threads using STL + for (int i = 0; i < numThreads; ++i) { + threads.emplace_back(cudaTask, i); + } + + // Wait for threads to finish + for (std::thread& t : threads) { + t.join(); + } + + // cudaTask(1); + + std::cout << "All threads have finished." << std::endl; + + return 0; +} + diff --git a/GPU-Virtual-Service/gpu-remoting/src/sample/testZeroCopy.cu b/GPU-Virtual-Service/gpu-remoting/src/sample/testZeroCopy.cu new file mode 100644 index 0000000..ab84f54 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/sample/testZeroCopy.cu @@ -0,0 +1,87 @@ +#include +#include +#include + +__global__ void vectorAddGPU(float *a, float *b, float *c, int N) +{ + int idx = blockIdx.x * blockDim.x + threadIdx.x; + + if (idx < N) + { + c[idx] = a[idx] + b[idx]; + } +} + +#define MEMORY_ALIGNMENT 4096 +#define ALIGN_UP(x, size) (((size_t)x + (size - 1)) & (~(size - 1))) + +int main(int argc, char **argv) +{ + int n, nelem; + unsigned int flags; + size_t bytes; + float *a, *b, *c; // Pinned memory allocated on the CPU + float *d_a, *d_b, *d_c; // Device pointers for mapped memory + float errorNorm, refNorm, ref, diff; + + + /* Allocate mapped CPU memory. */ + nelem = 1048576; + bytes = nelem * sizeof(float); + + flags = cudaHostAllocMapped; + assert(cudaSuccess == cudaHostAlloc((void **)&a, bytes, flags)); + assert(cudaSuccess == cudaHostAlloc((void **)&b, bytes, flags)); + assert(cudaSuccess == cudaHostAlloc((void **)&c, bytes, flags)); + printf("a: %p, b: %p, c: %p\n", a, b, c); + + /* Initialize the vectors. */ + for (n = 0; n < nelem; n++){ + a[n] = rand() / (float)RAND_MAX; + b[n] = rand() / (float)RAND_MAX; + } + + /* Get the device pointers for the pinned CPU memory mapped into the GPU + memory space. */ + assert(cudaSuccess == cudaHostGetDevicePointer((void **)&d_a, (void *)a, 0)); + assert(cudaSuccess == cudaHostGetDevicePointer((void **)&d_b, (void *)b, 0)); + assert(cudaSuccess == cudaHostGetDevicePointer((void **)&d_c, (void *)c, 0)); + printf("d_a: %p, d_b: %p, d_c: %p\n", d_a, d_b, d_c); + + + /* Call the GPU kernel using the CPU pointers residing in CPU mapped memory. */ + printf("> vectorAddGPU kernel will add vectors using mapped CPU memory...\n"); + dim3 block(256); + dim3 grid((unsigned int)ceil(nelem / (float)block.x)); + vectorAddGPU<<>>(d_a, d_b, d_c, nelem); + + cudaDeviceSynchronize(); + + /* Compare the results */ + printf("> Checking the results from vectorAddGPU() ...\n"); + errorNorm = 0.f; + refNorm = 0.f; + for (n = 0; n < nelem; n++){ + ref = a[n] + b[n]; + diff = c[n] - ref; + errorNorm += diff * diff; + refNorm += ref * ref; + if (n < 10) + printf("c[%d]=%f, ref=%f\n", n, c[n], ref); + } + + errorNorm = (float)sqrt((double)errorNorm); + refNorm = (float)sqrt((double)refNorm); + + + + /* Memory clean up */ + printf("> Releasing CPU memory...\n"); + assert(cudaSuccess == cudaFreeHost(a)); + assert(cudaSuccess == cudaFreeHost(b)); + assert(cudaSuccess == cudaFreeHost(c)); + + + + exit(errorNorm / refNorm < 1.e-6f ? EXIT_SUCCESS : EXIT_FAILURE); +} diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cuBlasCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cuBlasCallback.cc new file mode 100644 index 0000000..9220bf2 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cuBlasCallback.cc @@ -0,0 +1,185 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "cuBlasHandle"; + +DEFINE_SERVER_AM_CALLBACK(cublasCreate_v2Handle) { + tool::Logging(myName, "CUBLAS_CREATE_V2\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cublasHandle_t handle = NULL; + cublasStatus_t exit_code = cublasCreate_v2(&handle); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasCreate_v2 success, handle = %p\n", handle); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, handle); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLAS_CREATE_V2); + handle = (cublasHandle_t)serverEp->GetHandleVirAddr(handle, CUBLAS_CREATE_V2); + resBuf.Push64BitPointer(handle); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasCreate_v2 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasSgemm_v2Handle) { + tool::Logging(myName, "CUBLAS_SGEMM_V2\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cublasOperation_t transa = reqBuf.Pop(); + cublasOperation_t transb = reqBuf.Pop(); + int m = reqBuf.Pop(); + int n = reqBuf.Pop(); + int k = reqBuf.Pop(); + const float* alpha = reqBuf.AssignAddr(); + const float* A = (const float*)serverEp->GetDevPtr(reqBuf.Pop()); + int lda = reqBuf.Pop(); + const float* B = (const float*)serverEp->GetDevPtr(reqBuf.Pop()); + int ldb = reqBuf.Pop(); + const float* beta = reqBuf.AssignAddr(); + float* C = (float*)serverEp->GetDevPtr(reqBuf.Pop()); + int ldc = reqBuf.Pop(); + cublasStatus_t exit_code = cublasSgemm_v2(handle, transa, transb, m, n, k, + alpha, A, lda, B, ldb, beta, C, ldc); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasSgemm_v2 success, handle=%p\n", handle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasSgemm_v2 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasSgemmStridedBatchedHandle) { + tool::Logging(myName, "CUBLAS_SGEMM_STRIDED_BATCHED\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cublasOperation_t transa = reqBuf.Pop(); + cublasOperation_t transb = reqBuf.Pop(); + int m = reqBuf.Pop(); + int n = reqBuf.Pop(); + int k = reqBuf.Pop(); + const float* alpha = reqBuf.AssignAddr(); + const float* A = (const float*)serverEp->GetDevPtr(reqBuf.Pop()); + int lda = reqBuf.Pop(); + long long int strideA = reqBuf.Pop(); + const float* B = (const float*)serverEp->GetDevPtr(reqBuf.Pop()); + int ldb = reqBuf.Pop(); + long long int strideB = reqBuf.Pop(); + const float* beta = reqBuf.AssignAddr(); + float* C = (float*)serverEp->GetDevPtr(reqBuf.Pop()); + int ldc = reqBuf.Pop(); + long long int strideC = reqBuf.Pop(); + int batchCount = reqBuf.Pop(); + cublasStatus_t exit_code = cublasSgemmStridedBatched(handle, transa, transb, m, n, k, + alpha, A, lda, strideA, B, ldb, strideB, beta, + C, ldc, strideC, batchCount); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasSgemmStridedBatched success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasSgemmStridedBatched failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasDestroy_v2Handle) { + tool::Logging(myName, "CUBLAS_DESTROY_V2\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop(), true); + cublasStatus_t exit_code = cublasDestroy_v2(handle); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasDestroy_v2 success, handle = %p\n", handle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasDestroy_v2 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasSetStream_v2Handle) { + tool::Logging(myName, "CUBLAS_SET_STREAM_V2\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + cublasStatus_t exit_code = cublasSetStream_v2(handle, stream); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasSetStream_v2 success, handle=%p, stream=%p\n", handle, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasSetStream_v2 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasSetWorkspace_v2Handle) { + tool::Logging(myName, "CUBLAS_SET_WORKSPACE_V2\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop()); + void* workspace = serverEp->GetDevPtr(reqBuf.Pop()); + size_t size = reqBuf.Pop(); + cublasStatus_t exit_code = cublasSetWorkspace_v2(handle, workspace, size); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasSetWorkspace_v2 success, handle=%p\n", handle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasSetWorkspace_v2 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasSetMathModeHandle) { + tool::Logging(myName, "CUBLAS_SET_MATH_MODE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cublasMath_t mode = reqBuf.Pop(); + cublasStatus_t exit_code = cublasSetMathMode(handle, mode); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasSetMathMode success, handle=%p\n", handle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasSetMathMode failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasGetMathModeHandle) { + tool::Logging(myName, "CUBLAS_GET_MATH_MODE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasHandle_t handle = (cublasHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cublasMath_t mode; + cublasStatus_t exit_code = cublasGetMathMode(handle, &mode); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasGetMathMode success\n"); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLAS_GET_MATH_MODE); + resBuf.Push(mode); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasGetMathMode failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cuBlasLtCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cuBlasLtCallback.cc new file mode 100644 index 0000000..17d788e --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cuBlasLtCallback.cc @@ -0,0 +1,316 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "cuBlasLtHandle"; + +DEFINE_SERVER_AM_CALLBACK(cublasLtCreateHandle) { + tool::Logging(myName, "CUBLASLT_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cublasLtHandle_t lightHandle = NULL; + cublasStatus_t exit_code = cublasLtCreate(&lightHandle); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtCreate success, lightHandle = %p\n", lightHandle); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, lightHandle); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLASLT_CREATE); + lightHandle = (cublasLtHandle_t)serverEp->GetHandleVirAddr(lightHandle, CUBLASLT_CREATE); + resBuf.Push64BitPointer(lightHandle); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtCreate failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtDestroyHandle) { + tool::Logging(myName, "CUBLASLT_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtHandle_t lightHandle = (cublasLtHandle_t)serverEp->GetHandle(reqBuf.Pop(), true); + cublasStatus_t exit_code = cublasLtDestroy(lightHandle); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtDestroy success, lightHandle = %p\n", lightHandle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtDestroy failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulDescCreateHandle) { + tool::Logging(myName, "CUBLASLT_MATMULDESC_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cublasLtMatmulDesc_t matmulDesc; + cublasComputeType_t computeType = reqBuf.Pop(); + cudaDataType_t scaleType = reqBuf.Pop(); + cublasStatus_t exit_code = cublasLtMatmulDescCreate(&matmulDesc, computeType, scaleType); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulDescCreate success, matmulDesc = %p\n", matmulDesc); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, matmulDesc); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLASLT_MATMULDESC_CREATE); + matmulDesc = (cublasLtMatmulDesc_t)serverEp->GetHandleVirAddr(matmulDesc, CUBLASLT_MATMULDESC_CREATE); + resBuf.Push64BitPointer(matmulDesc); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulDescCreate failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulDescDestroyHandle) { + tool::Logging(myName, "CUBLASLT_MATMULDESC_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtMatmulDesc_t matmulDesc = (cublasLtMatmulDesc_t)serverEp->GetHandle(reqBuf.Pop(), true); + cublasStatus_t exit_code = cublasLtMatmulDescDestroy(matmulDesc); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulDescDestroy success, matmulDesc = %p\n", matmulDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulDescDestroy failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulDescSetAttributeHandle) { + tool::Logging(myName, "CUBLASLT_MATMULDESC_SETATTRIBUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtMatmulDesc_t matmulDesc = (cublasLtMatmulDesc_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatmulDescAttributes_t attr = reqBuf.Pop(); + const void* buf = reqBuf.AssignAddrForAll(); + size_t sizeInBytes = reqBuf.Pop(); + if (attr == CUBLASLT_MATMUL_DESC_BIAS_POINTER || attr == CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_POINTER + // || CUBLASLT_MATMUL_DESC_A_SCALE_POINTER || CUBLASLT_MATMUL_DESC_B_SCALE_POINTER || CUBLASLT_MATMUL_DESC_C_SCALE_POINTER || ... cuBlasLt v12.5 have more attributes related to devPtr + ) { + uint64_t* virtAddrPtr = (uint64_t*)buf; + void* devPtr = serverEp->GetDevPtr(*virtAddrPtr); + *virtAddrPtr = (uint64_t)devPtr; + } + + cublasStatus_t exit_code = cublasLtMatmulDescSetAttribute(matmulDesc, attr, buf, sizeInBytes); + + + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulDescSetAttribute success, desc = %p, attr = %d, buf = %p(val = %p), sizeInBytes = %lu\n", matmulDesc, attr, buf, *(uint64_t*)buf, sizeInBytes); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulDescSetAttribute failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatrixLayoutCreateHandle) { + tool::Logging(myName, "CUBLASLT_MATRIX_LAYOUT_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cublasLtMatrixLayout_t layout; + cudaDataType_t dataType = reqBuf.Pop(); + uint64_t rows = reqBuf.Pop(); + uint64_t cols = reqBuf.Pop(); + int64_t ld = reqBuf.Pop(); + cublasStatus_t exit_code = cublasLtMatrixLayoutCreate(&layout, dataType, rows, cols, ld); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatrixLayoutCreate success, layout = %p\n", layout); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, layout); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLASLT_MATRIX_LAYOUT_CREATE); + layout = (cublasLtMatrixLayout_t)serverEp->GetHandleVirAddr(layout, CUBLASLT_MATRIX_LAYOUT_CREATE); + resBuf.Push64BitPointer(layout); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatrixLayoutCreate failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatrixLayoutDestroyHandle) { + tool::Logging(myName, "CUBLASLT_MATRIX_LAYOUT_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtMatrixLayout_t layout = (cublasLtMatrixLayout_t)serverEp->GetHandle(reqBuf.Pop(), true); + cublasStatus_t exit_code = cublasLtMatrixLayoutDestroy(layout); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatrixLayoutDestroy success, layout = %p\n", layout); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatrixLayoutDestroy failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatrixLayoutSetAttributeHandle) { + tool::Logging(myName, "CUBLASLT_MATRIX_LAYOUT_SETATTRIBUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtMatrixLayout_t layout = (cublasLtMatrixLayout_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatrixLayoutAttribute_t attr = reqBuf.Pop(); + const void* buf = reqBuf.AssignAddrForAll(); + size_t sizeInBytes = reqBuf.Pop(); + cublasStatus_t exit_code = cublasLtMatrixLayoutSetAttribute(layout, attr, buf, sizeInBytes); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatrixLayoutSetAttribute success: layout = %p, attr = %d, buf = %p(val = %p), sizeInBytes = %lu\n", layout, attr, buf, *(uint64_t*)buf, sizeInBytes); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatrixLayoutSetAttribute failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulPreferenceCreateHandle) { + tool::Logging(myName, "CUBLASLT_MATMULPREFERENCE_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cublasLtMatmulPreference_t pref; + cublasStatus_t exit_code = cublasLtMatmulPreferenceCreate(&pref); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulPreferenceCreate success, pref = %p\n", pref); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, pref); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLASLT_MATMULPREFERENCE_CREATE); + pref = (cublasLtMatmulPreference_t)serverEp->GetHandleVirAddr(pref, CUBLASLT_MATMULPREFERENCE_CREATE); + resBuf.Push64BitPointer(pref); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulPreferenceCreate failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulPreferenceDestroyHandle) { + tool::Logging(myName, "CUBLASLT_MATMULPREFERENCE_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtMatmulPreference_t pref = (cublasLtMatmulPreference_t)serverEp->GetHandle(reqBuf.Pop(), true); + cublasStatus_t exit_code = cublasLtMatmulPreferenceDestroy(pref); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulPreferenceDestroy success, pref = %p\n", pref); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulPreferenceDestroy failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulPreferenceSetAttributeHandle) { + tool::Logging(myName, "CUBLASLT_MATMULPREFERENCE_SETATTRIBUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtMatmulPreference_t pref = (cublasLtMatmulPreference_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatmulPreferenceAttributes_t attr = reqBuf.Pop(); + const void* buf = reqBuf.AssignAddrForAll(); + size_t sizeInBytes = reqBuf.Pop(); + cublasStatus_t exit_code = cublasLtMatmulPreferenceSetAttribute(pref, attr, buf, sizeInBytes); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulPreferenceSetAttribute success: pref = %p, attr = %d, buf = %p(val = %p), sizeInBytes = %lu\n", pref, attr, buf, *(uint64_t*)buf, sizeInBytes); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulPreferenceSetAttribute failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulAlgoGetHeuristicHandle) { + tool::Logging(myName, "CUBLASLT_MATMULALGO_GETHEURISTIC\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtHandle_t lightHandle = (cublasLtHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatmulDesc_t matmulDesc = (cublasLtMatmulDesc_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatrixLayout_t layoutA = (cublasLtMatrixLayout_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatrixLayout_t layoutB = (cublasLtMatrixLayout_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatrixLayout_t layoutC = (cublasLtMatrixLayout_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatrixLayout_t layoutD = (cublasLtMatrixLayout_t)serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatmulPreference_t pref = (cublasLtMatmulPreference_t)serverEp->GetHandle(reqBuf.Pop()); + int reqAlgoCount = reqBuf.Pop(); + cublasLtMatmulHeuristicResult_t heuristicResult[reqAlgoCount]; + int retAlgoCount = 0; + cublasStatus_t exit_code = cublasLtMatmulAlgoGetHeuristic(lightHandle, matmulDesc, layoutA, layoutB, layoutC, layoutD, pref, reqAlgoCount, heuristicResult, &retAlgoCount); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmulAlgoGetHeuristic success: retAlgoCount = %d, lightHandle = %p, matmulDesc = %p, layoutA = %p, layoutB = %p, layoutC = %p, layoutD = %p, pref = %p\n", retAlgoCount, lightHandle, matmulDesc, layoutA, layoutB, layoutC, layoutD, pref); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUBLASLT_MATMULALGO_GETHEURISTIC); + resBuf.Push(retAlgoCount); + resBuf.Push(heuristicResult, retAlgoCount); + // heuristicResult contains cublasLtMatmulAlgo_t, which is a structure that describes the algorithm, and can be trivially serialized and later restored for use with the same version of cuBLAS library to save on selecting the right configuration again. + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmulAlgoGetHeuristic failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cublasLtMatmulHandle) { + tool::Logging(myName, "CUBLASLT_MATMUL\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cublasLtHandle_t lightHandle = (cublasLtHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatmulDesc_t compDesc = (cublasLtMatmulDesc_t) serverEp->GetHandle(reqBuf.Pop()); + const float* alpha = reqBuf.AssignAddr(); // default: host pointer, set by CUBLASLT_MATMUL_DESC_POINTER_MODE + const void* A = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cublasLtMatrixLayout_t Adesc = (cublasLtMatrixLayout_t) serverEp->GetHandle(reqBuf.Pop()); + const void* B = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cublasLtMatrixLayout_t Bdesc = (cublasLtMatrixLayout_t) serverEp->GetHandle(reqBuf.Pop()); + const float* beta = reqBuf.AssignAddr(); + const void* C = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cublasLtMatrixLayout_t Cdesc = (cublasLtMatrixLayout_t) serverEp->GetHandle(reqBuf.Pop()); + void* D = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cublasLtMatrixLayout_t Ddesc = (cublasLtMatrixLayout_t) serverEp->GetHandle(reqBuf.Pop()); + cublasLtMatmulAlgo_t* algo = reqBuf.AssignAddr(); + void* workspace = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + size_t workspaceSizeInBytes = reqBuf.Pop(); + cudaStream_t stream = (cudaStream_t) serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + cublasStatus_t exit_code = cublasLtMatmul(lightHandle, compDesc, alpha, A, Adesc, B, Bdesc, beta, C, Cdesc, D, Ddesc, algo, workspace, workspaceSizeInBytes, stream); + if (exit_code == CUBLAS_STATUS_SUCCESS) { + tool::Logging(myName, "cublasLtMatmul success\n"); + // cudaError_t exit_code_o = cudaDeviceSynchronize(); + // if (exit_code_o != cudaSuccess) { + // tool::Logging(LOG_ERROR, myName_, "cublasLtMatmul after cudaDeviceSynchronize failed: %d(%s)\n", exit_code_o, cudaGetErrorString(exit_code_o)); + // exit(EXIT_FAILURE); + // } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cublasLtMatmul failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cuDNNCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cuDNNCallback.cc new file mode 100644 index 0000000..7f6b7a4 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cuDNNCallback.cc @@ -0,0 +1,1179 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "cuDNNHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudnnCreateHandle) { + tool::Logging(myName, "CUDNN_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudnnHandle_t handle = NULL; + cudnnStatus_t exit_code = cudnnCreate(&handle); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnCreate success, handle = %p\n", handle); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, handle); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_CREATE); + handle = (cudnnHandle_t)serverEp->GetHandleVirAddr(handle, CUDNN_CREATE); + resBuf.Push64BitPointer(handle); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnCreate failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnDestroyHandle) { + tool::Logging(myName, "CUDNN_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudnnStatus_t exit_code = cudnnDestroy(handle); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnDestroy success, handle = %p\n", handle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnDestroy failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnCreateTensorDescriptorHandle) { + tool::Logging(myName, "CUDNN_CREATE_TENSOR_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudnnTensorDescriptor_t tensorDesc = NULL; + cudnnStatus_t exit_code = cudnnCreateTensorDescriptor(&tensorDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnCreateTensorDescriptor success, tensorDesc = %p\n", tensorDesc); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, tensorDesc); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_CREATE_TENSOR_DESCRIPTOR); + tensorDesc = (cudnnTensorDescriptor_t)serverEp->GetHandleVirAddr(tensorDesc, CUDNN_CREATE_TENSOR_DESCRIPTOR); + resBuf.Push64BitPointer(tensorDesc); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnCreateTensorDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnDestroyTensorDescriptorHandle) { + tool::Logging(myName, "CUDNN_DESTROY_TENSOR_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorDescriptor_t tensorDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudnnStatus_t exit_code = cudnnDestroyTensorDescriptor(tensorDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnDestroyTensorDescriptor success, tensorDesc = %p\n", tensorDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnDestroyTensorDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetTensorSizeInBytesHandle) { + tool::Logging(myName, "CUDNN_GET_TENSOR_SIZE_IN_BYTES\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorDescriptor_t tensorDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetTensorSizeInBytes(tensorDesc, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetTensorSizeInBytes success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_TENSOR_SIZE_IN_BYTES); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetTensorSizeInBytes failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetTensor4dDescriptorHandle) { + tool::Logging(myName, "CUDNN_SET_TENSOR_4D_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorDescriptor_t tensorDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorFormat_t format = reqBuf.Pop(); + cudnnDataType_t dataType = reqBuf.Pop(); + int n = reqBuf.Pop(); + int c = reqBuf.Pop(); + int h = reqBuf.Pop(); + int w = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnSetTensor4dDescriptor(tensorDesc, format, dataType, n, c, h, w); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetTensor4dDescriptor success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetTensor4dDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetTensorNdDescriptorHandle) { + tool::Logging(myName, "CUDNN_SET_TENSOR_ND_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorDescriptor_t tensorDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnDataType_t dataType = reqBuf.Pop(); + int nbDims = reqBuf.Pop(); + const int* dimA = reqBuf.AssignAddr(); + const int* strideA = reqBuf.AssignAddr(); + cudnnStatus_t exit_code = cudnnSetTensorNdDescriptor(tensorDesc, dataType, nbDims, dimA, strideA); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetTensorNdDescriptor success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetTensorNdDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetTensorNdDescriptorExHandle) { + tool::Logging(myName, "CUDNN_SET_TENSOR_ND_DESCRIPTOR_EX\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorDescriptor_t tensorDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorFormat_t format = reqBuf.Pop(); + cudnnDataType_t dataType = reqBuf.Pop(); + int nbDims = reqBuf.Pop(); + const int* dimA = reqBuf.AssignAddr(); + cudnnStatus_t exit_code = cudnnSetTensorNdDescriptorEx(tensorDesc, format, dataType, nbDims, dimA); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetTensorNdDescriptorEx success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetTensorNdDescriptorEx failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnCreateTensorTransformDescriptorHandle) { + tool::Logging(myName, "CUDNN_CREATE_TENSOR_TRANSFORM_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudnnTensorTransformDescriptor_t transformDesc = NULL; + cudnnStatus_t exit_code = cudnnCreateTensorTransformDescriptor(&transformDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnCreateTensorTransformDescriptor success, transformDesc = %p\n", transformDesc); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, transformDesc); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_CREATE_TENSOR_TRANSFORM_DESCRIPTOR); + transformDesc = (cudnnTensorTransformDescriptor_t)serverEp->GetHandleVirAddr(transformDesc, CUDNN_CREATE_TENSOR_TRANSFORM_DESCRIPTOR); + resBuf.Push64BitPointer(transformDesc); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnCreateTensorTransformDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetTensorTransformDescriptorHandle) { + tool::Logging(myName, "CUDNN_SET_TENSOR_TRANSFORM_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorTransformDescriptor_t transformDesc = (cudnnTensorTransformDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const uint32_t nbDims = reqBuf.Pop(); + const cudnnTensorFormat_t destFormat = reqBuf.Pop(); + const int32_t* padBeforeA = reqBuf.AssignAddr(); + const int32_t* padAfterA = reqBuf.AssignAddr(); + const uint32_t* foldA = reqBuf.AssignAddr(); // spatial dimension (dimensions 2 and up) + const cudnnFoldingDirection_t dirt = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnSetTensorTransformDescriptor(transformDesc, nbDims, destFormat, padBeforeA, padAfterA, foldA, dirt); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetTensorTransformDescriptor success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetTensorTransformDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnDestroyTensorTransformDescriptorHandle) { + tool::Logging(myName, "CUDNN_DESTROY_TENSOR_TRANSFORM_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorTransformDescriptor_t transformDesc = (cudnnTensorTransformDescriptor_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudnnStatus_t exit_code = cudnnDestroyTensorTransformDescriptor(transformDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnDestroyTensorTransformDescriptor success, transformDesc = %p\n", transformDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnDestroyTensorTransformDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnInitTransformDestHandle) { + tool::Logging(myName, "CUDNN_INIT_TRANSFORM_DEST\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnTensorTransformDescriptor_t transformDesc = (cudnnTensorTransformDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t srcDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t destDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + size_t destSizeInBytes = 0; + cudnnStatus_t exit_code = cudnnInitTransformDest(transformDesc, srcDesc, destDesc, &destSizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnInitTransformDest success, destSizeInBytes = %lu, transformDesc = %p, srcDesc = %p, destDesc = %p\n", destSizeInBytes, transformDesc, srcDesc, destDesc); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_INIT_TRANSFORM_DEST); + resBuf.Push(destSizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnInitTransformDest failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } + +} + +DEFINE_SERVER_AM_CALLBACK(cudnnTransformTensorExHandle) { + tool::Logging(myName, "CUDNN_TRANSFORM_TENSOR_EX\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorTransformDescriptor_t transDesc + = (const cudnnTensorTransformDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const void* alpha = reqBuf.AssignAddr(); + const cudnnTensorDescriptor_t srDesc= (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* srcData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* beta = reqBuf.AssignAddr(); + const cudnnTensorDescriptor_t dsDesc= (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* destData = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnTransformTensorEx(handle, transDesc, alpha, srDesc, srcData, beta, dsDesc, destData); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnTransformTensorEx success, handle = %p, transDesc = %p, srDesc = %p, dsDesc = %p\n", handle, transDesc, srDesc, dsDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnTransformTensorEx failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnTransformFilterHandle) { + tool::Logging(myName, "CUDNN_TRANSFORM_FILTER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorTransformDescriptor_t transDesc + = (const cudnnTensorTransformDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const void* alpha = reqBuf.AssignAddr(); + const cudnnFilterDescriptor_t srDesc= (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* srcData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* beta = reqBuf.AssignAddr(); + const cudnnFilterDescriptor_t dsDesc= (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* destData = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnTransformFilter(handle, transDesc, alpha, srDesc, srcData, beta, dsDesc, destData); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnTransformFilter success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnTransformFilter failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnCreateFilterDescriptorHandle) { + tool::Logging(myName, "CUDNN_CREATE_FILTER_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudnnFilterDescriptor_t filterDesc = NULL; + cudnnStatus_t exit_code = cudnnCreateFilterDescriptor(&filterDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnCreateFilterDescriptor success, filterDesc = %p\n", filterDesc); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, filterDesc); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_CREATE_FILTER_DESCRIPTOR); + filterDesc = (cudnnFilterDescriptor_t)serverEp->GetHandleVirAddr(filterDesc, CUDNN_CREATE_FILTER_DESCRIPTOR); + resBuf.Push64BitPointer(filterDesc); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnCreateFilterDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetFilterNdDescriptorHandle) { + tool::Logging(myName, "CUDNN_SET_FILTER_ND_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnFilterDescriptor_t filterDesc = (cudnnFilterDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnDataType_t dataType = reqBuf.Pop(); + cudnnTensorFormat_t format = reqBuf.Pop(); + int nbDims = reqBuf.Pop(); + const int* filterDimA = reqBuf.AssignAddr(); + cudnnStatus_t exit_code = cudnnSetFilterNdDescriptor(filterDesc, dataType, format, nbDims, filterDimA); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetFilterNdDescriptor success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetFilterNdDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnDestroyFilterDescriptorHandle) { + tool::Logging(myName, "CUDNN_DESTROY_FILTER_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnFilterDescriptor_t filterDesc = (cudnnFilterDescriptor_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudnnStatus_t exit_code = cudnnDestroyFilterDescriptor(filterDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnDestroyFilterDescriptor success, filterDesc = %p\n", filterDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnDestroyFilterDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetFilterSizeInBytesHandle) { + tool::Logging(myName, "CUDNN_GET_FILTER_SIZE_IN_BYTES\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnFilterDescriptor_t filterDesc = (cudnnFilterDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + size_t sizeInBytes= 0; + cudnnStatus_t exit_code = cudnnGetFilterSizeInBytes(filterDesc, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetFilterSizeInBytes success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_FILTER_SIZE_IN_BYTES); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetFilterSizeInBytes failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetFoldedConvBackwardDataDescriptorsHandle) { + tool::Logging(myName, "CUDNN_GET_FOLDED_CONV_BACKWARD_DATA_DESCRIPTORS\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnFilterDescriptor_t filterDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t diffDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t gradDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorFormat_t transformFormat = reqBuf.Pop(); + cudnnFilterDescriptor_t foldedFilterDesc = (cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t paddedDiffDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionDescriptor_t foldedConvDesc = (cudnnConvolutionDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t foldedGradDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorTransformDescriptor_t filterFoldTransDesc = (cudnnTensorTransformDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorTransformDescriptor_t diffPadTransDesc = (cudnnTensorTransformDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorTransformDescriptor_t gradFoldTransDesc = (cudnnTensorTransformDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorTransformDescriptor_t gradUnfoldTransDesc = (cudnnTensorTransformDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnGetFoldedConvBackwardDataDescriptors(handle, filterDesc, diffDesc, convDesc, gradDesc, transformFormat, foldedFilterDesc, paddedDiffDesc, foldedConvDesc, foldedGradDesc, filterFoldTransDesc, diffPadTransDesc, gradFoldTransDesc, gradUnfoldTransDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetFoldedConvBackwardDataDescriptors success, handle = %p, filterDesc = %p, diffDesc = %p, convDesc = %p, gradDesc = %p, transformFormat = %d, foldedFilterDesc = %p, paddedDiffDesc = %p, foldedConvDesc = %p, foldedGradDesc = %p, filterFoldTransDesc = %p, diffPadTransDesc = %p, gradFoldTransDesc = %p, gradUnfoldTransDesc = %p\n", handle, filterDesc, diffDesc, convDesc, gradDesc, transformFormat, foldedFilterDesc, paddedDiffDesc, foldedConvDesc, foldedGradDesc, filterFoldTransDesc, diffPadTransDesc, gradFoldTransDesc, gradUnfoldTransDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetFoldedConvBackwardDataDescriptors failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetStreamHandle) { + tool::Logging(myName, "CUDNN_SET_STREAM\n"); + ServerEndpoint* serverEp= (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t) serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + cudnnStatus_t exit_code = cudnnSetStream(handle, stream); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetStream success, handle = %p, stream = %p\n", handle, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetStream failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBatchNormalizationBackwardExHandle) { + tool::Logging(myName, "CUDNN_BATCH_NORMALIZATION_BACKWARD_EX\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBatchNormMode_t mode = reqBuf.Pop(); + cudnnBatchNormOps_t bnOps = reqBuf.Pop(); + const void* alphaDataDiff = reqBuf.AssignAddr(); // always one element + const void* betaDataDiff = reqBuf.AssignAddr(); // always one element + const void* alphaParamDiff= reqBuf.AssignAddr(); // always one element + const void* betaParamDiff = reqBuf.AssignAddr(); // always one element + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* xData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t yDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* yData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t dyDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* dyData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t dzDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* dzData = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t dxDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* dxData = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t dBnScaleBiasDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const void* bnScaleData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* bnBiasData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + void* dBnScaleData= (void*) serverEp->GetDevPtr(reqBuf.Pop()); + void* dBnBiasData = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + double epsilon = reqBuf.Pop(); + const void* savedMean = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* savedInvVariance = (const void*)serverEp->GetDevPtr(reqBuf.Pop()); + cudnnActivationDescriptor_t activationDesc = (cudnnActivationDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + void* workSpace = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + size_t workSpaceSizeInBytes = reqBuf.Pop(); + void* reserveSpace= (void*) serverEp->GetDevPtr(reqBuf.Pop()); + size_t reserveSpaceSizeInBytes = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnBatchNormalizationBackwardEx(handle, mode, bnOps, + alphaDataDiff, betaDataDiff, alphaParamDiff, betaParamDiff, + xDesc, xData, yDesc, yData, dyDesc, dyData, dzDesc, dzData, dxDesc, dxData, + dBnScaleBiasDesc, bnScaleData, bnBiasData, dBnScaleData, dBnBiasData, + epsilon, savedMean, savedInvVariance, activationDesc, + workSpace, workSpaceSizeInBytes, reserveSpace, reserveSpaceSizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBatchNormalizationBackwardEx success, handle = %p, xDesc = %p, yDesc = %p, dyDesc = %p, dzDesc = %p, dxDesc = %p, dBnScaleBiasDesc = %p\n", handle, xDesc, yDesc, dyDesc, dzDesc, dxDesc, dBnScaleBiasDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBatchNormalizationBackwardEx failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBatchNormalizationForwardTrainingExHandle) { + tool::Logging(myName, "CUDNN_BATCH_NORMALIZATION_FORWARD_TRAINING_EX\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBatchNormMode_t mode = reqBuf.Pop(); + cudnnBatchNormOps_t bnOps = reqBuf.Pop(); + const float* alphaData = reqBuf.AssignAddr(); // always one element + const float* betaData = reqBuf.AssignAddr(); // always one element + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* xData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t zDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* zData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t yDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* yData = serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t bnScaleBiasMeanVarDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* bnScaleData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* bnBiasData = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + double exponentialAverageFactor = reqBuf.Pop(); + void* resultRunningMean = serverEp->GetDevPtr(reqBuf.Pop()); + void* resultRunningVariance = serverEp->GetDevPtr(reqBuf.Pop()); + double epsilon = reqBuf.Pop(); + void* saveMean = serverEp->GetDevPtr(reqBuf.Pop()); + void* saveInvVari = serverEp->GetDevPtr(reqBuf.Pop()); + cudnnActivationDescriptor_t activationDesc = (cudnnActivationDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* workSpace = serverEp->GetDevPtr(reqBuf.Pop()); + size_t workSpaceSizeInBytes = reqBuf.Pop(); + void* reserveSpace= serverEp->GetDevPtr(reqBuf.Pop()); + size_t reserveSpaceSizeInBytes = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnBatchNormalizationForwardTrainingEx(handle, mode, bnOps, alphaData, betaData, xDesc, xData, zDesc, zData, yDesc, yData, bnScaleBiasMeanVarDesc, bnScaleData, bnBiasData, exponentialAverageFactor, resultRunningMean, resultRunningVariance, epsilon, saveMean, saveInvVari, activationDesc, workSpace, workSpaceSizeInBytes, reserveSpace, reserveSpaceSizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBatchNormalizationForwardTrainingEx success, handle = %p, xDesc = %p, zDesc = %p, yDesc = %p, bnScaleBiasMeanVarDesc = %p\n", handle, xDesc, zDesc, yDesc, bnScaleBiasMeanVarDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBatchNormalizationForwardTrainingEx failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBatchNormalizationForwardInferenceHandle) { + tool::Logging(myName, "CUDNN_BATCH_NORMALIZATION_FORWARD_INFERENCE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBatchNormMode_t mode = reqBuf.Pop(); + const float* alpha = reqBuf.AssignAddr(); // always one element + const float* beta = reqBuf.AssignAddr(); // always one element + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* x = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t yDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* y = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnTensorDescriptor_t bnScaleBiasMeanVarDesc = (cudnnTensorDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const void* bnScale = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* bnBias = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* estimatedMean = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const void* estimatedVariance = (const void*)serverEp->GetDevPtr(reqBuf.Pop()); + double epsilon = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnBatchNormalizationForwardInference(handle, mode, alpha, beta, xDesc, x, yDesc, y, bnScaleBiasMeanVarDesc, bnScale, bnBias, estimatedMean, estimatedVariance, epsilon); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBatchNormalizationForwardInference success, handle = %p, xDesc = %p, yDesc = %p, bnScaleBiasMeanVarDesc = %p\n", handle, xDesc, yDesc, bnScaleBiasMeanVarDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBatchNormalizationForwardInference failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBackendCreateDescriptorHandle) { + tool::Logging(myName, "CUDNN_BACKEND_CREATE_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnBackendDescriptorType_t decType = reqBuf.Pop(); + uint64_t virAddr = reqBuf.Pop(); + cudnnBackendDescriptor_t backendDesc = NULL; + cudnnStatus_t exit_code = cudnnBackendCreateDescriptor(decType, &backendDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBackendCreateDescriptor success, backendDesc = %p\n", backendDesc); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, backendDesc); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_BACKEND_CREATE_DESCRIPTOR); + backendDesc = (cudnnBackendDescriptor_t)serverEp->GetHandleVirAddr(backendDesc, CUDNN_BACKEND_CREATE_DESCRIPTOR); + resBuf.Push64BitPointer(backendDesc); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBackendCreateDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBackendDestroyDescriptorHandle) { + tool::Logging(myName, "CUDNN_BACKEND_DESTROY_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnBackendDescriptor_t backendDesc = (cudnnBackendDescriptor_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudnnStatus_t exit_code = cudnnBackendDestroyDescriptor(backendDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBackendDestroyDescriptor success, backendDesc = %p\n", backendDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBackendDestroyDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBackendSetAttributeHandle) { // todo: to be validated + tool::Logging(myName, "CUDNN_BACKEND_SET_ATTRIBUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnBackendDescriptor_t backendDesc = (cudnnBackendDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnBackendAttributeName_t attrName = reqBuf.Pop(); + cudnnBackendAttributeType_t attrType = reqBuf.Pop(); + int64_t eleCount = reqBuf.Pop(); + const uint8_t* arrayOfElem = reqBuf.AssignAddrForAll(); // size is eleCount * sizeof(...) + + if (attrType == CUDNN_TYPE_BACKEND_DESCRIPTOR || attrType == CUDNN_TYPE_HANDLE + || attrName == CUDNN_ATTR_EXECUTION_PLAN_COMPUTED_INTERMEDIATE_UIDS + || attrName == CUDNN_ATTR_EXECUTION_PLAN_RUN_ONLY_INTERMEDIATE_UIDS + || attrName == CUDNN_ATTR_INTERMEDIATE_INFO_UNIQUE_ID + || attrName == CUDNN_ATTR_INTERMEDIATE_INFO_DEPENDENT_DATA_UIDS + || attrName == CUDNN_ATTR_TENSOR_UNIQUE_ID + || attrName == CUDNN_ATTR_VARIANT_PACK_UNIQUE_IDS + || attrName == CUDNN_ATTR_LAYOUT_INFO_TENSOR_UID + ) { + uint64_t* ptrList = (uint64_t*)arrayOfElem; + for (int i = 0; i < eleCount; i++) { + ptrList[i] = (uint64_t)serverEp->GetHandle(ptrList[i]); + } + } + + if (attrType == CUDNN_TYPE_VOID_PTR) { + uint64_t* ptrList = (uint64_t*)arrayOfElem; + for (int i = 0; i < eleCount; i++) { + ptrList[i] = (uint64_t)serverEp->GetDevPtr(ptrList[i]); + } + } + + cudnnStatus_t exit_code = cudnnBackendSetAttribute(backendDesc, attrName, attrType, eleCount, arrayOfElem); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBackendSetAttribute success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBackendSetAttribute failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBackendGetAttributeHandle) { // todo: to be validated + tool::Logging(myName, "CUDNN_BACKEND_GET_ATTRIBUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnBackendDescriptor_t backendDesc = (cudnnBackendDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnBackendAttributeName_t attrName = reqBuf.Pop(); + cudnnBackendAttributeType_t attrType = reqBuf.Pop(); + int64_t reqElemCount = reqBuf.Pop(); + size_t varLen = reqBuf.Pop(); + // uint8_t* arrayOfElem = (uint8_t*)malloc(varLen * reqElemCount); + uint8_t arrayOfElem[varLen * reqElemCount]; + int64_t elementCount = 0; + cudnnStatus_t exit_code = cudnnBackendGetAttribute(backendDesc, attrName, attrType, reqElemCount, &elementCount, arrayOfElem); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBackendSetAttribute success\n"); + + if (attrType == CUDNN_TYPE_BACKEND_DESCRIPTOR || attrType == CUDNN_TYPE_HANDLE + || attrName == CUDNN_ATTR_EXECUTION_PLAN_COMPUTED_INTERMEDIATE_UIDS + || attrName == CUDNN_ATTR_EXECUTION_PLAN_RUN_ONLY_INTERMEDIATE_UIDS + || attrName == CUDNN_ATTR_INTERMEDIATE_INFO_UNIQUE_ID + || attrName == CUDNN_ATTR_INTERMEDIATE_INFO_DEPENDENT_DATA_UIDS + || attrName == CUDNN_ATTR_TENSOR_UNIQUE_ID + || attrName == CUDNN_ATTR_VARIANT_PACK_UNIQUE_IDS + || attrName == CUDNN_ATTR_LAYOUT_INFO_TENSOR_UID + ) { + uint64_t* ptrList = (uint64_t*)arrayOfElem; + for (int i = 0; i < elementCount; i++) { + ptrList[i] = serverEp->FindHandleVirAddr((void*)ptrList[i]); + } + } + + if (attrType == CUDNN_TYPE_VOID_PTR) { + uint64_t* ptrList = (uint64_t*)arrayOfElem; + for (int i = 0; i < elementCount; i++) { + ptrList[i] = serverEp->FindDevPtrVirAddr((void*)ptrList[i]); + } + } + + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_BACKEND_GET_ATTRIBUTE); + resBuf.Push(elementCount); + resBuf.Push(arrayOfElem, varLen * reqElemCount); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBackendSetAttribute failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBackendExecuteHandle) { + tool::Logging(myName, "CUDNN_BACKEND_EXECUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBackendDescriptor_t execPlan = (cudnnBackendDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnBackendDescriptor_t varianPack = (cudnnBackendDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnBackendExecute(handle, execPlan, varianPack); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBackendExecute success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBackendExecute failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnBackendFinalizeHandle) { + tool::Logging(myName, "CUDNN_BACKEND_FINALIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnBackendDescriptor_t descriptor = (cudnnBackendDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnBackendFinalize(descriptor); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnBackendFinalize success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnBackendFinalize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetBatchNormalizationBackwardExWorkspaceSizeHandle) { + tool::Logging(myName, "CUDNN_GET_BATCH_NORMALIZATION_BACKWARD_EX_WORKSPACE_SIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBatchNormMode_t mode = reqBuf.Pop(); + cudnnBatchNormOps_t bnOps = reqBuf.Pop(); + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t yDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dyDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dzDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dxDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dBnScaleBiasDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnActivationDescriptor_t activationDesc = (cudnnActivationDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetBatchNormalizationBackwardExWorkspaceSize(handle, mode, bnOps, xDesc, yDesc, dyDesc, dzDesc, dxDesc, dBnScaleBiasDesc, activationDesc, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetBatchNormalizationBackwardExWorkspaceSize success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_BATCH_NORMALIZATION_BACKWARD_EX_WORKSPACE_SIZE); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetBatchNormalizationBackwardExWorkspaceSize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetBatchNormalizationForwardTrainingExWorkspaceSizeHandle) { + tool::Logging(myName, "CUDNN_GET_BATCH_NORMALIZATION_FORWARD_TRAINING_EX_WORKSPACE_SIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBatchNormMode_t mode = reqBuf.Pop(); + cudnnBatchNormOps_t bnOps = reqBuf.Pop(); + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t zDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t yDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t bnScaleBiasMeanVarDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnActivationDescriptor_t activationDesc = (cudnnActivationDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize(handle, mode, bnOps, xDesc, zDesc, yDesc, bnScaleBiasMeanVarDesc, activationDesc, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_BATCH_NORMALIZATION_FORWARD_TRAINING_EX_WORKSPACE_SIZE); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetBatchNormalizationTrainingExReserveSpaceSizeHandle) { + tool::Logging(myName, "CUDNN_GET_BATCH_NORMALIZATION_TRAINING_EX_RESERVE_SPACE_SIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnBatchNormMode_t mode = reqBuf.Pop(); + cudnnBatchNormOps_t bnOps = reqBuf.Pop(); + cudnnActivationDescriptor_t activationDesc = (cudnnActivationDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetBatchNormalizationTrainingExReserveSpaceSize(handle, mode, bnOps, activationDesc, xDesc, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetBatchNormalizationTrainingExReserveSpaceSize success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_BATCH_NORMALIZATION_TRAINING_EX_RESERVE_SPACE_SIZE); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetBatchNormalizationTrainingExReserveSpaceSize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnCreateConvolutionDescriptorHandle) { + tool::Logging(myName, "CUDNN_CREATE_CONVOLUTION_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudnnConvolutionDescriptor_t convDesc = NULL; + cudnnStatus_t exit_code = cudnnCreateConvolutionDescriptor(&convDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnCreateConvolutionDescriptor success, convDesc = %p\n", convDesc); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, convDesc); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_CREATE_CONVOLUTION_DESCRIPTOR); + convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandleVirAddr(convDesc, CUDNN_CREATE_CONVOLUTION_DESCRIPTOR); + resBuf.Push64BitPointer(convDesc); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnCreateConvolutionDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnDestroyConvolutionDescriptorHandle) { + tool::Logging(myName, "CUDNN_DESTROY_CONVOLUTION_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudnnStatus_t exit_code = cudnnDestroyConvolutionDescriptor(convDesc); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnDestroyConvolutionDescriptor success, convDesc = %p\n", convDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnDestroyConvolutionDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetConvolutionGroupCountHandle) { + tool::Logging(myName, "CUDNN_SET_CONVOLUTION_GROUP_COUNT\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + int groupCount = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnSetConvolutionGroupCount(convDesc, groupCount); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetConvolutionGroupCount success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetConvolutionGroupCount failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetConvolutionMathTypeHandle) { + tool::Logging(myName, "CUDNN_SET_CONVOLUTION_MATH_TYPE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnMathType_t mathType = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnSetConvolutionMathType(convDesc, mathType); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetConvolutionMathType success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetConvolutionMathType failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetConvolutionNdDescriptorHandle) { + tool::Logging(myName, "CUDNN_SET_CONVOLUTION_ND_DESCRIPTOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + int arrayLength= reqBuf.Pop(); + const int* padA = reqBuf.AssignAddr(); + const int* filterStrideA = reqBuf.AssignAddr(); + const int* dilationA = reqBuf.AssignAddr(); + cudnnConvolutionMode_t mode = reqBuf.Pop(); + cudnnDataType_t dataType = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnSetConvolutionNdDescriptor(convDesc, arrayLength, padA, filterStrideA, dilationA, mode, dataType); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetConvolutionNdDescriptor success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetConvolutionNdDescriptor failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnSetConvolutionReorderTypeHandle) { + tool::Logging(myName, "CUDNN_SET_CONVOLUTION_REORDER_TYPE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnReorderType_t reorderType = reqBuf.Pop(); + cudnnStatus_t exit_code = cudnnSetConvolutionReorderType(convDesc, reorderType); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnSetConvolutionReorderType success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnSetConvolutionReorderType failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetConvolutionForwardAlgorithm_v7Handle) { + tool::Logging(myName, "CUDNN_GET_CONVOLUTION_FORWARD_ALGORITHM_V7\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnFilterDescriptor_t wDesc = (cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t yDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const int requestAlgoCount = reqBuf.Pop(); + int returnedAlgoCount = 0; + // cudnnConvolutionFwdAlgoPerf_t *perfResults = (cudnnConvolutionFwdAlgoPerf_t*)malloc(requestAlgoCount * sizeof(cudnnConvolutionFwdAlgoPerf_t)); + cudnnConvolutionFwdAlgoPerf_t perfResults[requestAlgoCount]; + cudnnStatus_t exit_code = cudnnGetConvolutionForwardAlgorithm_v7(handle, xDesc, wDesc, convDesc, yDesc, requestAlgoCount, &returnedAlgoCount, perfResults); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetConvolutionForwardAlgorithm_v7 success, returnedAlgoCount = %d\n", returnedAlgoCount); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_CONVOLUTION_FORWARD_ALGORITHM_V7); + resBuf.Push(returnedAlgoCount); + resBuf.Push(perfResults, requestAlgoCount); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetConvolutionForwardAlgorithm_v7 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetConvolutionBackwardFilterAlgorithm_v7Handle) { + tool::Logging(myName, "CUDNN_GET_CONVOLUTION_BACKWARD_FILTER_ALGORITHM_V7\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t xDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dyDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnFilterDescriptor_t dwDesc = (cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const int requestAlgoCount = reqBuf.Pop(); + int returnedAlgoCount = 0; + // cudnnConvolutionBwdFilterAlgoPerf_t *perfResults = (cudnnConvolutionBwdFilterAlgoPerf_t*)malloc(requestAlgoCount * sizeof(cudnnConvolutionBwdFilterAlgoPerf_t)); + cudnnConvolutionBwdFilterAlgoPerf_t perfResults[requestAlgoCount]; + cudnnStatus_t exit_code = cudnnGetConvolutionBackwardFilterAlgorithm_v7(handle, xDesc, dyDesc, convDesc, dwDesc, requestAlgoCount, &returnedAlgoCount, perfResults); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetConvolutionBackwardFilterAlgorithm_v7 success, returnedAlgoCount = %d\n", returnedAlgoCount); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_CONVOLUTION_BACKWARD_FILTER_ALGORITHM_V7); + resBuf.Push(returnedAlgoCount); + resBuf.Push(perfResults, requestAlgoCount); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetConvolutionBackwardFilterAlgorithm_v7 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetConvolutionBackwardDataAlgorithm_v7Handle) { + tool::Logging(myName, "CUDNN_GET_CONVOLUTION_BACKWARD_DATA_ALGORITHM_V7\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnFilterDescriptor_t wDesc = (cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dyDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionDescriptor_t convDesc = (cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnTensorDescriptor_t dxDesc = (cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const int requestAlgoCount = reqBuf.Pop(); + int returnedAlgoCount = 0; + // cudnnConvolutionBwdDataAlgoPerf_t *perfResults = (cudnnConvolutionBwdDataAlgoPerf_t*)malloc(requestAlgoCount * sizeof(cudnnConvolutionBwdDataAlgoPerf_t)); + cudnnConvolutionBwdDataAlgoPerf_t perfResults[requestAlgoCount]; + cudnnStatus_t exit_code = cudnnGetConvolutionBackwardDataAlgorithm_v7(handle, wDesc, dyDesc, convDesc, dxDesc, requestAlgoCount, &returnedAlgoCount, perfResults); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetConvolutionBackwardDataAlgorithm_v7 success, returnedAlgoCount = %d\n", returnedAlgoCount); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_CONVOLUTION_BACKWARD_DATA_ALGORITHM_V7); + resBuf.Push(returnedAlgoCount); + resBuf.Push(perfResults, requestAlgoCount); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetConvolutionBackwardDataAlgorithm_v7 failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetConvolutionForwardWorkspaceSizeHandle) { + tool::Logging(myName, "CUDNN_GET_CONVOLUTION_FORWARD_WORKSPACE_SIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t xDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnFilterDescriptor_t wDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t yDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionFwdAlgo_t algo = reqBuf.Pop(); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetConvolutionForwardWorkspaceSize(handle, xDesc, wDesc, convDesc, yDesc, algo, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetConvolutionForwardWorkspaceSize success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_CONVOLUTION_FORWARD_WORKSPACE_SIZE); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetConvolutionForwardWorkspaceSize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnConvolutionForwardHandle) { + tool::Logging(myName, "CUDNN_CONVOLUTION_FORWARD\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const void* alpha = reqBuf.AssignAddr(); // always one element + const cudnnTensorDescriptor_t xDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* x = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const cudnnFilterDescriptor_t wDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* w = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionFwdAlgo_t algo = reqBuf.Pop(); + void* workSpace = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + size_t workSpaceSizeInBytes = reqBuf.Pop(); + const void* beta = reqBuf.AssignAddr(); // always one element + const cudnnTensorDescriptor_t yDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* y = (void*) serverEp->GetDevPtr(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnConvolutionForward(handle, alpha, xDesc, x, wDesc, w, convDesc, algo, workSpace, workSpaceSizeInBytes, beta, yDesc, y); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnConvolutionForward success, handle = %p, xDesc = %p, wDesc = %p, convDesc = %p, yDesc = %p\n", handle, xDesc, wDesc, convDesc, yDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnConvolutionForward failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetConvolutionBackwardDataWorkspaceSizeHandle) { + tool::Logging(myName, "CUDNN_GET_CONVOLUTION_BACKWARD_DATA_WORKSPACE_SIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnFilterDescriptor_t wDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t dyDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t dxDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionBwdDataAlgo_t algo = reqBuf.Pop(); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetConvolutionBackwardDataWorkspaceSize(handle, wDesc, dyDesc, convDesc, dxDesc, algo, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetConvolutionBackwardDataWorkspaceSize success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_CONVOLUTION_BACKWARD_DATA_WORKSPACE_SIZE); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetConvolutionBackwardDataWorkspaceSize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnConvolutionBackwardFilterHandle) { + tool::Logging(myName, "CUDNN_CONVOLUTION_BACKWARD_FILTER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const void* alpha = reqBuf.AssignAddr(); // always one element + const cudnnTensorDescriptor_t xDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* x = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const cudnnTensorDescriptor_t dyDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* dy = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionBwdFilterAlgo_t algo = reqBuf.Pop(); + void* workSpace = serverEp->GetDevPtr(reqBuf.Pop()); + size_t workSpaceSizeInBytes = reqBuf.Pop(); + const void* beta = reqBuf.AssignAddr(); // always one element + const cudnnFilterDescriptor_t dwDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* dw = serverEp->GetDevPtr(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnConvolutionBackwardFilter(handle, alpha, xDesc, x, dyDesc, dy, convDesc, algo, workSpace, workSpaceSizeInBytes, beta, dwDesc, dw); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnConvolutionBackwardFilter success, handle = %p, xDesc = %p, dyDesc = %p, convDesc = %p, dwDesc = %p\n", handle, xDesc, dyDesc, convDesc, dwDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnConvolutionBackwardFilter failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnGetConvolutionBackwardFilterWorkspaceSizeHandle) { + tool::Logging(myName, "CUDNN_GET_CONVOLUTION_BACKWARD_FILTER_WORKSPACE_SIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t xDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnTensorDescriptor_t dyDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + const cudnnFilterDescriptor_t dwDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionBwdFilterAlgo_t algo = reqBuf.Pop(); + size_t sizeInBytes = 0; + cudnnStatus_t exit_code = cudnnGetConvolutionBackwardFilterWorkspaceSize(handle, xDesc, dyDesc, convDesc, dwDesc, algo, &sizeInBytes); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnGetConvolutionBackwardFilterWorkspaceSize success, sizeInBytes = %lu\n", sizeInBytes); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDNN_GET_CONVOLUTION_BACKWARD_FILTER_WORKSPACE_SIZE); + resBuf.Push(sizeInBytes); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnGetConvolutionBackwardFilterWorkspaceSize failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudnnConvolutionBackwardDataHandle) { + tool::Logging(myName, "CUDNN_CONVOLUTION_BACKWARD_DATA\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudnnHandle_t handle = (cudnnHandle_t) serverEp->GetHandle(reqBuf.Pop()); + const void* alpha = reqBuf.AssignAddr(); // always one element + const cudnnFilterDescriptor_t wDesc = (const cudnnFilterDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* w = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const cudnnTensorDescriptor_t dyDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + const void* dy = (const void*) serverEp->GetDevPtr(reqBuf.Pop()); + const cudnnConvolutionDescriptor_t convDesc = (const cudnnConvolutionDescriptor_t)serverEp->GetHandle(reqBuf.Pop()); + cudnnConvolutionBwdDataAlgo_t algo = reqBuf.Pop(); + void* workSpace = serverEp->GetDevPtr(reqBuf.Pop()); + size_t workSpaceSizeInBytes = reqBuf.Pop(); + const void* beta = reqBuf.AssignAddr(); // always one element + const cudnnTensorDescriptor_t dxDesc = (const cudnnTensorDescriptor_t) serverEp->GetHandle(reqBuf.Pop()); + void* dx = serverEp->GetDevPtr(reqBuf.Pop()); + cudnnStatus_t exit_code = cudnnConvolutionBackwardData(handle, alpha, wDesc, w, dyDesc, dy, convDesc, algo, workSpace, workSpaceSizeInBytes, beta, dxDesc, dx); + if (exit_code == CUDNN_STATUS_SUCCESS) { + tool::Logging(myName, "cudnnConvolutionBackwardData success, handle = %p, wDesc = %p, dyDesc = %p, convDesc = %p, dxDesc = %p\n", handle, wDesc, dyDesc, convDesc, dxDesc); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudnnConvolutionBackwardData failed: %d\n", exit_code); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtDeviceCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtDeviceCallback.cc new file mode 100644 index 0000000..4be4fd3 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtDeviceCallback.cc @@ -0,0 +1,155 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "CUDARuntimeDeviceHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudaGetDeviceHandle) { + tool::Logging(myName, "CUDA_GET_DEVICE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int device = 0; + + cudaError_t exit_code = cudaGetDevice(&device); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaGetDevice success, device = %d\n", device); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_GET_DEVICE); + resBuf.Push(device); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaGetDevice failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaGetDeviceCountHandle) { + tool::Logging(myName, "CUDA_GET_DEVICE_COUNT\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int count = 0; + + cudaError_t exit_code = cudaGetDeviceCount(&count); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaGetDeviceCount success, count = %d\n", count); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_GET_DEVICE_COUNT); + resBuf.Push(count); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaGetDeviceCount failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaGetDevicePropertiesHandle) { + tool::Logging(myName, "CUDA_GET_DEVICE_PROPERTIES\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int device = reqBuf.Pop(); + cudaDeviceProp prop; + + cudaError_t exit_code = cudaGetDeviceProperties(&prop, device); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaGetDeviceProperties success, device = %d\n", device); + prop.canMapHostMemory = 0; //! disable UVA + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_GET_DEVICE_PROPERTIES); + resBuf.Push(prop); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaGetDeviceProperties failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaDeviceGetAttributeHandle) { + tool::Logging(myName, "CUDA_DEVICE_GET_ATTRIBUTE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int value = 0; + cudaDeviceAttr attr = (cudaDeviceAttr)reqBuf.Pop(); + int device = reqBuf.Pop(); + + cudaError_t exit_code = cudaDeviceGetAttribute(&value, attr, device); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaDeviceGetAttribute success, device = %d, attr = %d, value = %d\n", device, attr, value); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_DEVICE_GET_ATTRIBUTE); + resBuf.Push(value); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaDeviceGetAttribute failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaSetDeviceHandle) { + tool::Logging(myName, "CUDA_SET_DEVICE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int deviceIdx = reqBuf.Pop(); //! index of devList, not device id + + cudaError_t exit_code = cudaSetDevice(deviceIdx); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaSetDevice success, device = %d\n", deviceIdx); + serverEp->curDev_ = deviceIdx; + serverEp->CliDev_ = deviceIdx; + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaSetDevice failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaSetMainDeviceHandle) { + tool::Logging(myName, "CUDA_SET_MAIN_DEVICE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int deviceIdx = reqBuf.Pop(); + serverEp->clientPID_ = reqBuf.Pop(); + serverEp->priority_ = reqBuf.Pop(); + serverEp->recoveryFlag_ = reqBuf.Pop(); + + serverEp->replayFlag_ = serverEp->recoveryFlag_; + cudaError_t exit_code = cudaSetDevice(deviceIdx); + if (exit_code == cudaSuccess) { + tool::Logging(LOG_INFO, serverEp->myName_, "cudaSetMainDevice success, curDev_ = %d, clientPID_ = %d, priority_ = %zu\n", deviceIdx, serverEp->clientPID_, serverEp->priority_); + serverEp->curDev_ = deviceIdx; + serverEp->CliDev_ = deviceIdx; + serverEp->backupFilePath_ = std::string(BACKUP_FILE_DIR) + "flexgv_backup_" + std::to_string(serverEp->clientID_) + "_" + std::to_string(serverEp->clientPID_) + ".dat"; + + // serverEp->SendStatus(123); + + if (serverEp->recoveryFlag_) { + serverEp->LoadFromStorage(); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaSetMainDevice failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaDeviceSynchronizeHandle) { + tool::Logging(myName, "CUDA_DEVICE_SYNCHRONIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaError_t exit_code = cudaDeviceSynchronize(); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaDeviceSynchronize success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaDeviceSynchronize failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtEventCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtEventCallback.cc new file mode 100644 index 0000000..b260b74 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtEventCallback.cc @@ -0,0 +1,161 @@ +/* CUDA Event API */ +#include "../../include/serverEndpoint.h" + +static const char* myName = "CUDARuntimeEventHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudaEventCreateHandle) { + tool::Logging(myName, "CUDA_EVENT_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudaEvent_t event = NULL; + cudaError_t exit_code = cudaEventCreate(&event); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaEventCreate success, event ptr: %p, size=%d\n", event, sizeof(uint64_t)); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, event); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_EVENT_CREATE); + event = (cudaEvent_t)serverEp->GetHandleVirAddr(event, CUDA_EVENT_CREATE); + resBuf.Push64BitPointer(event); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaEventCreate failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaEventCreateWithFlagsHandle) { + tool::Logging(myName, "CUDA_EVENT_CREATE_WITH_FLAGS\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudaEvent_t event = NULL; + unsigned int flags = reqBuf.Pop(); + cudaError_t exit_code = cudaEventCreateWithFlags(&event,flags); + if (exit_code == cudaSuccess) { + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, event); + tool::Logging(LOG_DEBUG, myName, "cudaEventCreateWithFlags success, event ptr: %p(virAddr=%p, pid=%d, ttid=%d), flags=%u\n", event, virAddr, serverEp->clientPID_, reqBuf.GetThreadID(), flags); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_EVENT_CREATE_WITH_FLAGS); + void* virtAddr = (cudaEvent_t)serverEp->GetHandleVirAddr(event, CUDA_EVENT_CREATE_WITH_FLAGS); + resBuf.Push64BitPointer(virtAddr); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + tool::Logging(LOG_DEBUG, myName, "cudaEventCreateWithFlags success, event ptr: %p(new virAddr=%p, pid=%d, ttid=%d), flags=%u\n", event, virtAddr, serverEp->clientPID_, reqBuf.GetThreadID(), flags); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaEventCreateWithFlags failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaEventRecordHandle){ + tool::Logging(myName, "CUDA_EVENT_Record\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t eventVirAddr= reqBuf.Pop(); + cudaEvent_t event = (cudaEvent_t)serverEp->GetHandle(eventVirAddr); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + cudaError_t exit_code = cudaEventRecord(event,stream); + + if (exit_code == cudaSuccess) { +#ifdef GV_HANDLE + Handle_t* handle = serverEp->GetHandleInfo(eventVirAddr); + if (handle != NULL) { + handle->stream = (uint64_t)stream; + } +#endif + + tool::Logging(myName, "cudaEventRecord success, eventPtr=%p, streamPtr=%p(isCommStream=%d)\n",event,stream,stream == serverEp->commStream_); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaEventRecord failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaEventQueryHandle) { + tool::Logging(myName, "CUDA_EVENT_QUERY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t eventVirAddr= reqBuf.Pop(); + cudaEvent_t event = (cudaEvent_t)serverEp->GetHandle(eventVirAddr); + cudaError_t exit_code = cudaSuccess; + if (event == NULL) { + tool::Logging(LOG_ERROR, myName, "cudaEventQuery warning: eventPtr is NULL(virAddr=%p, pid=%d, ttid=%d)\n", eventVirAddr, serverEp->clientPID_, reqBuf.GetThreadID()); + } + else { + exit_code = cudaEventQuery(event); + } + + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_EVENT_QUERY); + resBuf.Push(exit_code); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaEventQuery success, eventPtr=%p\n",event); + } + else if (exit_code == cudaErrorNotReady) { + tool::Logging(myName, "cudaEventQuery success, eventPtr=%p, cudaErrorNotReady(any captured work is incomplete)\n",event); + } + else { + tool::Logging(LOG_ERROR, myName, "cudaEventQuery failed for eventPtr=%p: %s\n", event, cudaGetErrorName(exit_code)); + } + return UCS_OK; +} + +DEFINE_SERVER_AM_CALLBACK(cudaEventDestroyHandle) { + tool::Logging(myName, "CUDA_EVENT_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t eventVirAddr= reqBuf.Pop(); + cudaEvent_t event = (cudaEvent_t)serverEp->GetHandle(eventVirAddr, true); + if (event == NULL) { + tool::Logging(LOG_ERROR, myName, "cudaEventDestroy success, eventPtr=%p(virAddr=%p, pid=%d, ttid=%d)\n",event, eventVirAddr, serverEp->clientPID_, reqBuf.GetThreadID()); + return UCS_OK; + } + cudaError_t exit_code = cudaEventDestroy(event); + if (exit_code == cudaSuccess) { + tool::Logging(LOG_DEBUG, myName, "cudaEventDestroy success, eventPtr=%p(virAddr=%p, pid=%d, ttid=%d)\n",event, eventVirAddr, serverEp->clientPID_, reqBuf.GetThreadID()); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaEventDestroy failed: %s, eventPtr=%p\n", cudaGetErrorName(exit_code), event); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaEventElapsedTimeHandle) { + tool::Logging(myName, "CUDA_EVENT_ELAPSED_TIME\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + float ms = 0; + cudaEvent_t start = (cudaEvent_t)serverEp->GetHandle(reqBuf.Pop()); + cudaEvent_t end = (cudaEvent_t)serverEp->GetHandle(reqBuf.Pop()); + cudaError_t exit_code = cudaEventElapsedTime(&ms,start,end); + if (exit_code == cudaSuccess || exit_code == cudaErrorNotReady) { + tool::Logging(myName, "cudaEventElapsedTime: %s, startPtr=%p, endPtr=%p, ms=%f\n",cudaGetErrorName(exit_code), start,end,ms); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_EVENT_ELAPSED_TIME); + resBuf.Push(exit_code); + resBuf.Push(ms); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaEventElapsedTime failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtExecutionCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtExecutionCallback.cc new file mode 100644 index 0000000..1f30448 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtExecutionCallback.cc @@ -0,0 +1,250 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "CUDARuntimeExecutionHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudaLaunchKernelHandle) { + tool::Logging(myName, "CUDA_LAUNCH_KERNEL\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t func = reqBuf.Pop(); + char* deviceName = reqBuf.AssignCString(); + dim3 gridDim = reqBuf.Pop(); + dim3 blockDim = reqBuf.Pop(); + uint8_t* paraValList = reqBuf.AssignAddrForAll(); + size_t sharedMem = reqBuf.Pop(); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + size_t paramNum = reqBuf.Pop(); + uint16_t* paramOffsets= reqBuf.AssignAddrForAll(); + uint16_t* paramSizes = reqBuf.AssignAddrForAll(); + + CUfunction cuFunc = NULL; + auto it = serverEp->_cuInfoMap.mapHost2CuFunc->find(func); + if (it == serverEp->_cuInfoMap.mapHost2CuFunc->end()) { + tool::Logging(LOG_ERROR, myName, "cudaLaunchKernel failed: unknown function(%p)\n", func); + return UCS_ERR_IO_ERROR; + } + else { + cuFunc = it->second; + } + +#ifdef GV_MEMORY_PTX + if (serverEp->_cuInfoMap.ptxExtractor->_finished == false) { + serverEp->_cuInfoMap.ptxExtractor->_readyClosed = true; + while (serverEp->_cuInfoMap.ptxExtractor->_finished == false) { + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + } + + std::vector>* dev_ptr_list = NULL; + auto newIt = serverEp->_cuInfoMap.mapDevName2DevPtr->find(deviceName); + if (newIt == serverEp->_cuInfoMap.mapDevName2DevPtr->end()) { + tool::Logging(LOG_ERROR, myName, "cudaLaunchKernel failed: unknown kernel name(%s)\n", deviceName); + exit(EXIT_FAILURE); + // return UCS_ERR_IO_ERROR; + } + else { + dev_ptr_list = (std::vector>*)newIt->second; + tool::Logging(myName, "cudaLaunchKernel: kernelName=%s, devPtrListSize=%zu\n", deviceName, dev_ptr_list->size()); + } +#endif // GV_MEMORY_PTX + +#ifdef GV_MEMORY + bool savedDevPtrList = true; + // std::vector>* dev_ptr_list = NULL; + std::set>* dev_ptr_set = NULL; + auto newIt = serverEp->_cuInfoMap.mapDevName2DevPtr->find(deviceName); + if (newIt == serverEp->_cuInfoMap.mapDevName2DevPtr->end()) { + // dev_ptr_list = new std::vector>(); + dev_ptr_set = new std::set>(); + serverEp->_cuInfoMap.mapDevName2DevPtr->insert({deviceName, (uint64_t)dev_ptr_set}); + savedDevPtrList = false; + } + else { + dev_ptr_set = (std::set>*)newIt->second; + tool::Logging(myName, "cudaLaunchKernel: kernelName=%s, devPtrListSize=%zu\n", deviceName, dev_ptr_set->size()); + savedDevPtrList = !(serverEp->_cuInfoMap.isFirstIter); + } +#endif // GV_MEMORY + + void** paramPtrList= (void**)malloc(paramNum * sizeof(void*)); + size_t ptrIdx = 0; + for (size_t i = 0; i < paramNum; i++) { + paramPtrList[i] = paraValList + paramOffsets[i]; + tool::Logging(myName, "param[%zu]: paramPtr=%p, val=%p, paramSize=%d\n", i, (uint64_t*)paramPtrList[i], *(uint64_t*)paramPtrList[i], paramSizes[i]); + +#ifdef GV_MEMORY + if (paramSizes[i] < sizeof(uint64_t)) { + continue; + } + if (!savedDevPtrList) { + size_t paramIntraOffset = 0; + while (paramIntraOffset <= paramSizes[i] - sizeof(uint64_t)){ + uint64_t* paramIntraPtr = (uint64_t*)((uint64_t)paramPtrList[i] + paramIntraOffset); + uint64_t tmpAddr = *paramIntraPtr; + tool::Logging(myName, "\t paramIntraOffset=%zu, tmpAddr=%p\n", paramIntraOffset, (void*)tmpAddr); + uint64_t realPtr = 0; + if (serverEp->_cuInfoMap.blockManager->FindByVirAddr(tmpAddr, realPtr) == -1) { + tool::Logging(myName, "\t warning - tmpAddr(%p) is not in blockManager\n", tmpAddr); + paramIntraOffset ++; + continue; + } + *paramIntraPtr = (uint64_t)realPtr; + tool::Logging(myName, "\t paramIntraOffset=%zu, devPtr=%p\n", paramIntraOffset, *paramIntraPtr); + dev_ptr_set->insert({i, paramIntraOffset}); + paramIntraOffset += sizeof(uint64_t); + } + } + else { + std::vector> dev_ptr_list(dev_ptr_set->begin(), dev_ptr_set->end()); + while (ptrIdx < dev_ptr_list.size()) { + if ((dev_ptr_list)[ptrIdx].first != i) { + break; + } + size_t paramIntraOffset = (dev_ptr_list)[ptrIdx].second; + uint64_t* paramIntraPtr = (uint64_t*)((uint64_t)paramPtrList[i] + paramIntraOffset); + uint64_t usrVirtAddr = *paramIntraPtr; + tool::Logging(LOG_REGS, myName, "\t paramIntraOffset=%zu, usrVirtAddr=%p\n", paramIntraOffset, usrVirtAddr); + uint64_t realPtr = 0; + ptrIdx++; + if (serverEp->_cuInfoMap.blockManager->FindByVirAddr(usrVirtAddr, realPtr) == -1) { + tool::Logging(LOG_REGS, myName, "\t warning - usrVirtAddr(%p) is not in blockManager\n", usrVirtAddr); + continue; + } + *paramIntraPtr = (uint64_t)realPtr; + tool::Logging(myName, "\t paramIntraOffset=%zu, devPtr=%p\n", paramIntraOffset, *paramIntraPtr); + } + } +#endif // GV_MEMORY + +#ifdef GV_MEMORY_PTX + if (strcmp(deviceName, "_ZN2at6native6sbtopk10gatherTopKIfjLi2ELb0EEEvNS_4cuda6detail10TensorInfoIT_T0_EES7_S7_bS7_S7_S8_S7_NS5_IlS7_EES7_PS6_" ) == 0 && i == 0) { + tool::Logging(myName, "param[%zu]: paramPtr=%p, val=%p, paramSize=%d\n", i, (uint64_t*)paramPtrList[i], *(uint64_t*)paramPtrList[i], paramSizes[i]); + size_t paramIntraOffset = 0; + uint64_t* paramIntraPtr = (uint64_t*)((uint64_t)paramPtrList[i] + paramIntraOffset); + uint64_t usrVirtAddr = *paramIntraPtr; // get the devPtr from current param #i + tool::Logging(myName, "\t paramIntraOffset=%zu, usrVirtAddr=%p\n", paramIntraOffset, usrVirtAddr); + if (usrVirtAddr == 0) { // the devPtr can be NULL + continue; + } + uint64_t blockID = GET_BLOCK_ID(usrVirtAddr); + uint64_t blockIntraOffset = GET_BLOCK_INTER_OFFSET(usrVirtAddr); + *paramIntraPtr = (uint64_t)serverEp->GetDevPtr(usrVirtAddr); + tool::Logging(myName, "\t blockID=%p, blockIntraOffset=%zu, devPtr=%p\n", blockID, blockIntraOffset, *paramIntraPtr); + } + + while (ptrIdx < dev_ptr_list->size()) { + if ((*dev_ptr_list)[ptrIdx].first != i) { // next devPtr is not in current param #i + break; + } + size_t paramIntraOffset = (*dev_ptr_list)[ptrIdx].second; + uint64_t* paramIntraPtr = (uint64_t*)((uint64_t)paramPtrList[i] + paramIntraOffset); + uint64_t usrVirtAddr = *paramIntraPtr; // get the devPtr from current param #i + tool::Logging(myName, "\t paramIntraOffset=%zu, usrVirtAddr=%p\n", paramIntraOffset, usrVirtAddr); + ptrIdx++; // ready move to next devPtr, which may be in the same param #i + if (usrVirtAddr == 0) { // the devPtr can be NULL + continue; + } + + uint64_t blockID = GET_BLOCK_ID(usrVirtAddr); + uint64_t blockIntraOffset = GET_BLOCK_INTER_OFFSET(usrVirtAddr); + if (blockID <= 0 || blockID >= serverEp->_cuInfoMap.blockInfoList.size()) { + tool::Logging(myName, "\t warning - blockID(%p), current blockNum=%zu\n", blockID, serverEp->_cuInfoMap.blockInfoList.size()); + // exit(EXIT_FAILURE); + continue; + } + if (blockIntraOffset >= serverEp->_cuInfoMap.blockInfoList[blockID].size) { + tool::Logging(myName, "\t cudaLaunchKernel: warning - blockIntraOffset(%p), current block size=%zu\n", blockIntraOffset, serverEp->_cuInfoMap.blockInfoList[blockID].size); + // exit(EXIT_FAILURE); + continue; + } + *paramIntraPtr = (uint64_t)serverEp->GetDevPtr(usrVirtAddr); + tool::Logging(myName, "\t blockID=%p, blockIntraOffset=%zu, devPtr=%p\n", blockID, blockIntraOffset, *paramIntraPtr); + } +#endif // GV_MEMORY_PTX + + } + + cudaSetDevice(serverEp->curDev_); // explicitly set the device and make sure the context is valid + + CUresult exit_code = cuLaunchKernel(cuFunc, gridDim.x, gridDim.y, gridDim.z, + blockDim.x, blockDim.y, blockDim.z, sharedMem, + (CUstream)stream, paramPtrList, NULL); + if (exit_code == CUDA_SUCCESS) { + tool::Logging(myName, "cudaLaunchKernel success: %s\n", deviceName); +#ifdef GV_MEMORY_PTX + cudaError_t sync_code = cudaDeviceSynchronize(); + if (sync_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaDeviceSynchronize (after launchKernel: %s) failed: %s\n", deviceName, cudaGetErrorName(sync_code)); + exit(EXIT_FAILURE); + } +#endif // GV_MEMORY_PTX + if (serverEp->curIter_ <= 1 // necessary to synchronize the non-blocking stream for the first iterations + || serverEp->recoveryFlag_ // synchronize the replaying kernel (e.g., indexSelectLargeIndex) + ) { + cudaError_t sync_code = cudaStreamSynchronize(stream); + if (sync_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaStreamSynchronize (after launchKernel: %s) failed: %s\n", deviceName, cudaGetErrorName(sync_code)); + exit(EXIT_FAILURE); + } + } + return UCS_OK; + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "cudaLaunchKernel failed: %s\n", errorStr); + // return UCS_ERR_IO_ERROR; + exit(EXIT_FAILURE); + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaFuncGetAttributesHandle) { + tool::Logging(myName, "CUDA_FUNC_GET_ATTRIBUTES\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + struct cudaFuncAttributes attr; + uint64_t func = reqBuf.Pop(); + + CUfunction cuFunc = NULL; + auto it = serverEp->_cuInfoMap.mapHost2CuFunc->find(func); + if (it == serverEp->_cuInfoMap.mapHost2CuFunc->end()) { + tool::Logging(LOG_ERROR, myName, "cudaLaunchKernel failed: unknown function(%p)\n", func); + return UCS_ERR_IO_ERROR; + } + else { + cuFunc = it->second; + } + + int exit_code = CUDA_SUCCESS; + int shrSzBytes = 0; + int conSzBytes = 0; + int locSzBytes = 0; + exit_code |= cuFuncGetAttribute(&shrSzBytes, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, cuFunc); + exit_code |= cuFuncGetAttribute(&conSzBytes, CU_FUNC_ATTRIBUTE_CONST_SIZE_BYTES, cuFunc); + exit_code |= cuFuncGetAttribute(&locSzBytes, CU_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, cuFunc); + attr.sharedSizeBytes = shrSzBytes; + attr.constSizeBytes = conSzBytes; + attr.localSizeBytes = locSzBytes; + exit_code |= cuFuncGetAttribute((int*)&attr.maxThreadsPerBlock, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, cuFunc); + exit_code |= cuFuncGetAttribute((int*)&attr.numRegs, CU_FUNC_ATTRIBUTE_NUM_REGS, cuFunc); + exit_code |= cuFuncGetAttribute((int*)&attr.ptxVersion, CU_FUNC_ATTRIBUTE_PTX_VERSION, cuFunc); + exit_code |= cuFuncGetAttribute((int*)&attr.binaryVersion, CU_FUNC_ATTRIBUTE_BINARY_VERSION, cuFunc); + exit_code |= cuFuncGetAttribute((int*)&attr.cacheModeCA, CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, cuFunc); + exit_code |= cuFuncGetAttribute((int*)&attr.maxDynamicSharedSizeBytes, CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, cuFunc); + exit_code |= cuFuncGetAttribute((int*)&attr.preferredShmemCarveout, CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, cuFunc); + + + if (exit_code == CUDA_SUCCESS) { + tool::Logging(myName, "cudaFuncGetAttributes success\n"); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_FUNC_GET_ATTRIBUTES); + resBuf.Push(attr); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaFuncGetAttributes failed\n"); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtInternalCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtInternalCallback.cc new file mode 100644 index 0000000..5fc7dfd --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtInternalCallback.cc @@ -0,0 +1,319 @@ +#include "../../include/serverEndpoint.h" + +#include "../../include/hook/fatBinary.h" + +extern "C" { +extern void **__cudaRegisterFatBinary(void *fatCubin); +extern void __cudaRegisterFatBinaryEnd(void **fatCubinHandle); +extern void __cudaUnregisterFatBinary(void **fatCubinHandle); +extern void __cudaRegisterFunction(void **fatCubinHandle, const char *hostFun, + char *deviceFun, const char *deviceName, + int thread_limit, uint3 *tid, uint3 *bid, + dim3 *bDim, dim3 *gDim, int *wSize); +extern void __cudaRegisterVar(void **fatCubinHandle, char *hostVar, + char *deviceAddress, const char *deviceName, + int ext, size_t size, int constant, int global); +extern void __cudaRegisterSharedVar(void **fatCubinHandle, void **devicePtr, + size_t size, size_t alignment, int storage); +extern void __cudaRegisterShared(void **fatCubinHandle, void **devicePtr); +// extern void __cudaRegisterTexture(void **fatCubinHandle, +// const textureReference *hostVar, +// void **deviceAddress, char *deviceName, +// int dim, int norm, int ext); +// extern void __cudaRegisterSurface(void **fatCubinHandle, +// const surfaceReference *hostVar, +// void **deviceAddress, char *deviceName, +// int dim, int ext); +} + +static const char* myName = "CUDARuntimeInternalHandle"; + +DEFINE_SERVER_AM_CALLBACK(__cudaRegisterHandle) { + tool::Logging(LOG_REGS, myName, "__CUDA_REGISTER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RegisterIOV reqBuf = RegisterIOV(header, header_length, data); + size_t reqNum = reqBuf.GetRequestNum(); + CUmodule curModule = NULL; + + if (!serverEp->initFlag_) { + cudaSetDevice(serverEp->curDev_); + cudaFree(0); // explicitly initialize the context for the current device + + int streamPriority = -1 * (int)serverEp->priority_; + cudaError_t exit_code = cudaStreamCreateWithPriority(&serverEp->defaultStream_, cudaStreamNonBlocking, streamPriority); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaStreamCreate failed: %s\n", cudaGetErrorString(exit_code)); + exit(EXIT_FAILURE); + } + else { + tool::Logging(LOG_REGS, myName, "serverEp defaultStream = %p\n", serverEp->defaultStream_); + } + // serverEp->defaultStream_ = NULL; + + tool::Logging(myName, "initFlag_ is false\n"); + for (int i = 0; i < 3; i++) { + serverEp->shmQueSizes_[i] = new SharedMemoryOpt(("/flexgv_shm_"+std::to_string(serverEp->clientID_)+ "_" + std::to_string(serverEp->clientPID_) + "_" + std::to_string(i + 1)).c_str(), sizeof(size_t) * 3, false); // dataFeeder may be the first one to create the shared memory and initialize the space + } + +#ifdef GV_MEMORY_PTX + serverEp->_cuInfoMap.imageQueue = new moodycamel::BlockingReaderWriterQueue< std::pair >(240); + serverEp->_cuInfoMap.ptxExtractor = new PTXExtractor(serverEp->_cuInfoMap.imageQueue, serverEp->_cuInfoMap.mapDevName2DevPtr); + boost::thread_attributes attrs; + attrs.set_stack_size(THREAD_STACK_SIZE); + serverEp->_ptxThread = new boost::thread(attrs, boost::bind(&PTXExtractor::Run, serverEp->_cuInfoMap.ptxExtractor)); +#endif // GV_MEMORY_PTX + + serverEp->initFlag_ = true; + } + else { + tool::Logging(myName, "initFlag_ is true\n"); + } + + for (size_t i = 0; i < reqNum; i++) { + int reqType = reqBuf.Pop(); + switch (reqType) { + case __CUDA_REGISTER_FAT_BINARY: { + uint64_t fatBinHandle = reqBuf.Pop(); + size_t fatBinSize = reqBuf.Pop(); + size_t kernelNum = reqBuf.Pop(); + void* fatBinText = reqBuf.AssignAddr(); + void* image = fatBinText; +#ifdef GV_MEMORY_PTX + image = malloc(fatBinSize); + memcpy(image, fatBinText, fatBinSize); + serverEp->_fatbinList.push_back(image); + + if (serverEp->_cuInfoMap.imageQueue){ + serverEp->_cuInfoMap.imageQueue->enqueue(std::make_pair(image, fatBinSize)); + } +#endif // GV_MEMORY_PTX + + CUresult exit_code = cuModuleLoadData(&curModule, image); + if (exit_code == CUDA_SUCCESS) { + serverEp->_cuInfoMap.mapFatBinHandle2CuModule->insert({fatBinHandle, curModule}); + tool::Logging(LOG_REGS, myName, "__cudaRegisterFatBinary success: loaded the module(%p) from fatCubin(%p) text\n", curModule, fatBinHandle); + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "__cudaRegisterFatBinary failed: %s\n", errorStr); + return UCS_ERR_IO_ERROR; + } + break; + } + case __CUDA_REGISTER_FUNCTION: { + uint64_t fatBinHandler = reqBuf.Pop(); + const char* hostFun = (const char *)reqBuf.Pop(); + const char* deviceName = strdup(reqBuf.AssignCString()); + size_t paramNum = reqBuf.Pop(); + CUfunction cuFunc = NULL; + CUresult exit_code = cuModuleGetFunction(&cuFunc, curModule, deviceName); + if (exit_code == CUDA_SUCCESS) { + serverEp->_cuInfoMap.mapHost2CuFunc->insert({(uint64_t)hostFun, cuFunc}); + tool::Logging(LOG_REGS, myName, "__cudaRegisterFunction success: registered function(%s) with %zu parameters\n", deviceName, paramNum); + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "__cudaRegisterFunction failed for function(%s): %s\n", deviceName, errorStr); + return UCS_ERR_IO_ERROR; + } + break; + } + case __CUDA_REGISTER_VAR: { + uint64_t fatBinHandler = reqBuf.Pop(); + char* hostVar = (char*)reqBuf.Pop(); + const char* deviceName = strdup(reqBuf.AssignCString()); + CUdeviceptr devPtr = 0; + size_t bytes = 0; + CUresult exit_code = cuModuleGetGlobal(&devPtr, &bytes, curModule, deviceName); + if (exit_code == CUDA_SUCCESS) { + serverEp->_cuInfoMap.mapHostVar2CuDevPtr->insert({(uint64_t)hostVar, devPtr}); + tool::Logging(LOG_REGS, myName, "__cudaRegisterVar success: registered variable(hostVar: %p, devName: %s, devPtr: %p)\n", hostVar, deviceName, devPtr); + } + else { + const char* errorStr; + tool::Logging(LOG_ERROR, myName, "__cudaRegisterVar failed for variable(%s): %s\n", deviceName, errorStr); + return UCS_ERR_IO_ERROR; + } + break; + } + } + } + tool::Logging(LOG_DEBUG, myName, "__cudaRegister success: registered %zu items, %zu bytes in total\n", reqNum, length); + return UCS_OK; +} + +DEFINE_SERVER_AM_CALLBACK(__cudaRegisterFatBinaryHandle){ + tool::Logging(LOG_REGS, myName, "__CUDA_REGISTER_FAT_BINARY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t fatBinHandle= reqBuf.Pop(); + size_t fatBinSize = reqBuf.Pop(); + size_t kernelNum = reqBuf.Pop(); + void* fatBinText = reqBuf.AssignAddr(); + void* image = NULL; + image = malloc(fatBinSize); + memcpy(image, fatBinText, fatBinSize); + +#ifdef GV_MEMORY_PTX + if (serverEp->_cuInfoMap.imageQueue){ + serverEp->_cuInfoMap.imageQueue->enqueue(std::make_pair(image, fatBinSize)); + } +#endif // GV_MEMORY_PTX + + cudaSetDevice(serverEp->curDev_); + + + cudaFree(0); // explicitly initialize the context for device #i + + CUmodule cuModule = NULL; + CUresult exit_code = cuModuleLoadData(&cuModule, image); + if (exit_code == CUDA_SUCCESS) { + tool::Logging(LOG_REGS, myName, "__cudaRegisterFatBinary success: loaded the module(%p) from fatCubin(%p) text in device #%d\n", cuModule, fatBinHandle, serverEp->curDev_); + serverEp->_cuInfoMap.mapFatBinHandle2CuModule->insert({fatBinHandle, cuModule}); + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "__cudaRegisterFatBinary failed in device #%d: %s\n", serverEp->curDev_, errorStr); + return UCS_ERR_IO_ERROR; + } + + // delete[] fatBinText; + cudaSetDevice(serverEp->curDev_); + + if (param->recv_attr & UCP_AM_RECV_ATTR_FLAG_RNDV) { + free(image); + } + + return UCS_OK; +} + +DEFINE_SERVER_AM_CALLBACK(__cudaRegisterFunctionHandle) { + tool::Logging(LOG_REGS, myName, "__CUDA_REGISTER_FUNCTION\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t fatBinHandler = reqBuf.Pop(); + const char* hostFun = (const char *)reqBuf.Pop(); + const char* deviceName = strdup(reqBuf.AssignCString()); + size_t paramNum = reqBuf.Pop(); + + cudaSetDevice(serverEp->curDev_); + auto it = serverEp->_cuInfoMap.mapFatBinHandle2CuModule->find(fatBinHandler); + if (it == serverEp->_cuInfoMap.mapFatBinHandle2CuModule->end()) { + tool::Logging(LOG_ERROR, myName, "__cudaRegisterFunction failed: unknown handler(%p) in device #%d\n", fatBinHandler, serverEp->curDev_); + return UCS_ERR_IO_ERROR; + } + CUmodule cuModule = it->second; + CUfunction cuFunc = NULL; + CUresult exit_code = cuModuleGetFunction(&cuFunc, cuModule, deviceName); + if (exit_code == CUDA_SUCCESS) { + serverEp->_cuInfoMap.mapHost2CuFunc->insert({(uint64_t)hostFun, cuFunc}); + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "__cudaRegisterFunction failed for function(%s) in device #%d: %s\n", deviceName, serverEp->curDev_, errorStr); + return UCS_ERR_IO_ERROR; + } + + tool::Logging(LOG_REGS, myName, "__cudaRegisterFunction success: registered function(%s) with %zu parameters\n", deviceName, paramNum); + cudaSetDevice(serverEp->curDev_); + return UCS_OK; +} + +void ServerEndpoint::__cudaUnregisterFatBinaryHandle() { + tool::Logging(myName, "__CUDA_UNREGISTER_FAT_BINARY\n"); + + for (auto& pair : *_cuInfoMap.mapFatBinHandle2CuModule) { + CUresult exit_code = cuModuleUnload(pair.second); + if (exit_code != CUDA_SUCCESS) { + tool::Logging(LOG_ERROR, myName, "__cudaUnregisterFatBinary failed: can not unload the module(%p) from fatCubin(%p) text\n", pair.second, pair.first); + } + } + for (int i = 0; i < _fatbinList.size(); i++) { + free(_fatbinList[i]); + } + for (auto& pair : *_cuInfoMap.mapDevName2DevPtr) { + auto tmpSet = reinterpret_cast>*>(pair.second); + delete tmpSet; + } + _cuInfoMap.mapFatBinHandle2CuModule->clear(); + _cuInfoMap.mapHost2CuFunc->clear(); + _cuInfoMap.mapHostVar2CuDevPtr->clear(); + +#ifdef GV_MEMORY + const std::vector& blocks = _cuInfoMap.blockManager->GetBlocks(); + for (int i = 0; i < blocks.size(); i++) { + if (blocks[i].valid == false) { // maybe have already been freed + continue; + } + cudaError_t exit_code = cudaFree((void*)blocks[i].devPtr); + _cuInfoMap.blockManager->ResetBlock(0, i); + } + delete _cuInfoMap.blockManager; +#else + for (size_t blockIdx = 0; blockIdx < _cuInfoMap.blockInfoList.size(); blockIdx++) { + if (_cuInfoMap.blockInfoList[blockIdx].valid == false) { + continue; + } + cudaError_t exit_code = cudaFree((void*)_cuInfoMap.blockInfoList[blockIdx].devPtr); // maybe have already been freed + _cuInfoMap.blockInfoList[blockIdx].valid = false; + } +#endif + + tool::Logging(LOG_DEBUG, myName, "__cudaUnregisterFatBinary success: unloaded all the modules and cleared all the maps of client #%llu\n", clientID_); + // tool::Logging(LOG_INFO, myName, "blockInfoList size: %zu, handleInfoList size: %zu (valid size: %zu)\n", _cuInfoMap.blockInfoList.size(), _cuInfoMap.handleManager->GetCapacity(), _cuInfoMap.handleManager->GetValidHandleNum()); + + cudaStreamDestroy(defaultStream_); + +#ifdef GV_BACKUP + StopBackup(); + StopCommEventMonitor(); +#endif // GV_BACKUP + +#ifdef GV_HANDLE + delete _cuInfoMap.handleManager; +#endif // GV_HANDLE + + delete _cuInfoMap.mapFatBinHandle2CuModule; + delete _cuInfoMap.mapHost2CuFunc; + delete _cuInfoMap.mapHostVar2CuDevPtr; + + delete _cuInfoMap.ptxExtractor; // ptxExtractor will clean up the imageQueue, mapDevName2DevPtr + delete _ptxThread; +} + +DEFINE_SERVER_AM_CALLBACK(__cudaRegisterVarHandle) { + tool::Logging(LOG_REGS, myName, "__CUDA_REGISTER_VAR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t fatBinHandler = reqBuf.Pop(); + + char* hostVar = (char*)reqBuf.Pop(); + const char* deviceName = strdup(reqBuf.AssignCString()); + + cudaSetDevice(serverEp->curDev_); + + auto it = serverEp->_cuInfoMap.mapFatBinHandle2CuModule->find(fatBinHandler); + if (it == serverEp->_cuInfoMap.mapFatBinHandle2CuModule->end()) { + tool::Logging(LOG_ERROR, myName, "__cudaRegisterVar failed: unknown handler(%p) in device #%d\n", fatBinHandler, serverEp->curDev_); + return UCS_ERR_IO_ERROR; + } + CUmodule cuModule = it->second; + CUdeviceptr devPtr = 0; + size_t bytes = 0; + CUresult exit_code = cuModuleGetGlobal(&devPtr, &bytes, cuModule, deviceName); + if (exit_code == CUDA_SUCCESS) { + tool::Logging(LOG_REGS, myName, "__cudaRegisterVar success: registered variable(hostVar: %p, devName: %s, devPtr: %p in device #%d)\n", hostVar, deviceName, devPtr, serverEp->curDev_); + serverEp->_cuInfoMap.mapHostVar2CuDevPtr->insert({(uint64_t)hostVar, devPtr}); + } + else { + tool::Logging(LOG_ERROR, myName, "__cudaRegisterVar failed: can not load the variable in device #%d\n", serverEp->curDev_); + return UCS_ERR_IO_ERROR; + } + + cudaSetDevice(serverEp->curDev_); + return UCS_OK; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtMemoryCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtMemoryCallback.cc new file mode 100644 index 0000000..00f1bb5 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtMemoryCallback.cc @@ -0,0 +1,467 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "CUDARuntimeMemoryHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudaMallocHandle){ + tool::Logging(myName, "CUDA_MALLOC\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + size_t cuda_size = reqBuf.Pop(); + bool essential = reqBuf.Pop(); + + void* devPtr = NULL; + cudaError_t exit_code = cudaMalloc(&devPtr, cuda_size); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMalloc success, devPtr = %p with size = %d\n", devPtr, cuda_size); + uint64_t virtAddr = (uint64_t)devPtr; + +#ifdef GV_MEMORY + virtAddr = serverEp->_cuInfoMap.blockManager->AddBlock((uint64_t)devPtr, cuda_size, essential); +#else + size_t blockNum = ( + serverEp->_cuInfoMap.blockInfoList.emplace_back(Block_t{.devPtr = (uint64_t)devPtr, .size = cuda_size, .valid = true}), + serverEp->_cuInfoMap.blockInfoList.size() - 1 + ); +#endif // GV_MEMORY + +#ifdef GV_MEMORY_PTX + if (blockNum >= BLOCKS_MAX_NUM) { + tool::Logging(LOG_ERROR, myName, "cudaMalloc failed: current blockNum(%zu) >= BLOCKS_MAX_NUM(%zu)\n", blockNum, BLOCKS_MAX_NUM); + return UCS_ERR_IO_ERROR; + } + virtAddr = blockNum << BLOCK_SHIFT_BIT; +#endif // GV_MEMORY_PTX + +#ifdef GV_Scheduler + serverEp->sche->cal_job_mem(cuda_size); +#endif // GV_Scheduler + + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_MALLOC); + // resBuf.Push64BitPointer(devPtr); + resBuf.Push(virtAddr); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMalloc failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemsetHandle) { + tool::Logging(myName, "CUDA_MEMSET\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + void* devPtr = serverEp->GetDevPtr(reqBuf.Pop()); + // void* devPtr = (void*)reqBuf->Pop(); + int value = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + + cudaError_t exit_code = cudaMemset(devPtr, value, count); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemset success, devPtr = %p(%llu) with value = %d and count = %d\n", devPtr, devPtr, value, count); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemset failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemsetAsyncHandle) { + tool::Logging(myName, "CUDA_MEMSET_ASYNC\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + void* devPtr = serverEp->GetDevPtr(reqBuf.Pop()); + // void* devPtr = (void*)reqBuf->Pop(); + int value = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + + cudaError_t exit_code = cudaMemsetAsync(devPtr, value, count, stream); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemset success, devPtr = %p(%llu) with value = %d , count = %d , streamPtr=%p\n", devPtr, devPtr, value, count, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemset failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemGetInfoHandle) { + tool::Logging(myName, "CUDA_MEM_GET_INFO\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int replayDone = reqBuf.Pop(); + size_t free = 0; + size_t total = 0; + cudaError exit_code = cudaMemGetInfo(&free,&total); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemGetInfo success, free = %lu, total= %lu\n",free,total); + if (replayDone) { + serverEp->recoveryFlag_ = false; + serverEp->_cuInfoMap.handleManager->Shrink(); + tool::Logging(LOG_INFO, "Backup", "replay done.\n"); + } + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_MEM_GET_INFO); + resBuf.Push(free); + resBuf.Push(total); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemGetInfo failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaFreeHandle) { + tool::Logging(myName, "CUDA_FREE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virtAddr = reqBuf.Pop(); + void* devPtr = serverEp->GetDevPtr(virtAddr); + + cudaError_t exit_code = cudaFree(devPtr); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaFree success, devPtr = %p\n", devPtr); +#ifdef GV_MEMORY_PTX + size_t blockIdx = serverEp->GetBlockIdx(virtAddr); + serverEp->_cuInfoMap.blockInfoList[blockIdx].valid = false; +#endif // GV_MEMORY_PTX + +#ifdef GV_MEMORY + serverEp->_cuInfoMap.blockManager->ResetBlock(virtAddr); +#endif // GV_MEMORY + + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaFree failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyH2DHandle) { + tool::Logging(myName, "CUDA_MEMCPY_H2D\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + cudaMemcpyKind kind = cudaMemcpyHostToDevice; + size_t count = 0; + void* dst = NULL; + ucs_status_t status = UCS_OK; + uint8_t* headers = (uint8_t*)header; // header contains parameters for cudaMemcpy H2D + memcpy(&kind, headers, sizeof(cudaMemcpyKind)); + memcpy(&count, headers + sizeof(cudaMemcpyKind), sizeof(size_t)); + memcpy(&dst, headers + sizeof(cudaMemcpyKind) + sizeof(size_t), sizeof(uint64_t)); + dst = serverEp->GetDevPtr((uint64_t)dst); + + if (kind != cudaMemcpyHostToDevice || count != length) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpy H2D failed: invalid cudaMemcpyKind(%d) or count(%zu)\n", kind, count); + return UCS_ERR_IO_ERROR; + } + if (param->recv_attr & UCP_AM_RECV_ATTR_FLAG_RNDV) { + tool::Logging(myName, "cudaMemcpyH2D: RNDV request.\n"); + Request_t recv_request_ctx = {.type = 1, .complete = 0}; + ucp_request_param_t param2; + param2.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | + UCP_OP_ATTR_FIELD_DATATYPE | + UCP_OP_ATTR_FIELD_USER_DATA| + UCP_OP_ATTR_FIELD_MEMORY_TYPE; + param2.op_attr_mask |= UCP_OP_ATTR_FLAG_NO_IMM_CMPL; + param2.datatype = ucp_dt_make_contig(1); + param2.user_data = &recv_request_ctx; + param2.cb.recv_am = (ucp_am_recv_data_nbx_callback_t)RecvCallBack; + param2.memory_type = UCS_MEMORY_TYPE_CUDA; + Request_t* rndv_request = (Request_t*)ucp_am_recv_data_nbx(serverEp->_dataWorker, data, dst, count, ¶m2); + status = Wait(rndv_request, &recv_request_ctx, &serverEp->_dataWorker); + if (status == UCS_OK) { + tool::Logging(myName, "cudaMemcpy H2D: Host -> Device(%p) with %zu B\n", dst, count); + } + else { + tool::Logging(myName, "cudaMemcpy H2D failed: ucp_am_recv_data_nbx(%s)\n", ucs_status_string(status)); + } + ucp_request_free(rndv_request); + return status; + } + else { + tool::Logging(myName, "cudaMemcpy H2D: Eager request.\n"); + cudaError_t exit_code = cudaMemcpy(dst, data, count, kind); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemcpy H2D: Host -> Device(%p) with %zu B\n", dst, count); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpy H2D failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyD2HHandle) { + tool::Logging(myName, "CUDA_MEMCPY_D2H\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaMemcpyKind kind = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + void* src = serverEp->GetDevPtr(reqBuf.Pop()); + + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_MEMCPY_D2H); + resBuf.Push((uint8_t*)src, count); + serverEp->SendResponse(&resBuf, ¶m->reply_ep, UCS_MEMORY_TYPE_CUDA); + + tool::Logging(myName, "cudaMemcpy D2H: Device(%p) -> Host with %zu B\n", src, count); + return UCS_OK; +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyD2DHandle) { + tool::Logging(myName, "CUDA_MEMCPY_D2D\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaMemcpyKind kind = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + void* src = serverEp->GetDevPtr(reqBuf.Pop()); + void* dst = serverEp->GetDevPtr(reqBuf.Pop()); + + cudaError_t exit_code = cudaMemcpy(dst, src, count, kind); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemcpy D2D: Device(%p) -> Device(%p) with %zu B\n", src, dst, count); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpy D2D failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyAsyncH2DHandle) { + tool::Logging(myName, "CUDA_MEMCPY_ASYNC_H2D\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + cudaMemcpyKind kind = cudaMemcpyHostToDevice; + size_t count = 0; + cudaStream_t stream = NULL; + uint8_t CurType = MEMCPY_OTHER; + void* dst = NULL; + ucs_status_t status = UCS_OK; + uint8_t* headers = (uint8_t*)header; // header contains parameters for cudaMemcpyAsync H2D + size_t headerOffset= 0; + memcpy(&kind, headers + headerOffset, sizeof(cudaMemcpyKind)); + headerOffset += sizeof(cudaMemcpyKind); + memcpy(&count, headers + headerOffset, sizeof(size_t)); + headerOffset += sizeof(size_t); + memcpy(&stream,headers + headerOffset, sizeof(uint64_t)); + stream = (cudaStream_t)serverEp->GetHandle((uint64_t)stream); + headerOffset += sizeof(uint64_t); + memcpy(&CurType,headers + headerOffset, sizeof(uint8_t)); + headerOffset += sizeof(uint8_t); + memcpy(&dst, headers + headerOffset, sizeof(uint64_t)); + headerOffset += sizeof(uint64_t); + dst = serverEp->GetDevPtr((uint64_t)dst); + + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + + if (kind != cudaMemcpyHostToDevice) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync H2D failed: invalid cudaMemcpyKind(%d)\n", kind); + return UCS_ERR_IO_ERROR; + } + + if (CurType != MEMCPY_OTHER) { // with data preloading optimization, data is already in shared memory queue + size_t shmQueueSize= 0; + size_t shmQueueIden= 0; + if (serverEp->shmQueues_[CurType - 1] == NULL) { + if (serverEp->shmQueSizes_[CurType - 1]->ReadCurSize(&shmQueueSize) == false || shmQueueSize == 0) { + tool::Logging(LOG_ERROR, myName, "failed: read shared memory queue size failed\n"); + // return UCS_ERR_IO_ERROR; + exit(EXIT_FAILURE); + } +#ifdef GV_Scheduler + if(serverEp->shmQueSizes_[CurType - 1]->ReadCurNumIt(&(serverEp->numIterations)) == false){ + tool::Logging(LOG_ERROR, myName, "failed: read numIterations failed\n", kind); + return UCS_ERR_IO_ERROR; + } + serverEp->sche->get_Iteration(serverEp->numIterations); + tool::Logging(LOG_INFO, myName, "numIterations=%d\n", serverEp->numIterations); +#endif + if (serverEp->shmQueSizes_[CurType - 1]->ReadCurIdent(&shmQueueIden) == false) { + tool::Logging(LOG_ERROR, myName, "failed: read shared memory queue identify prefix failed\n"); + // return UCS_ERR_IO_ERROR; + exit(EXIT_FAILURE); + } + tool::Logging(myName, "cudaMemcpyAsync H2D: shmQueueSize=%zu, shmQueueIdentifyPrefix=%zu\n", shmQueueSize, shmQueueIden); + serverEp->shmQueues_[CurType - 1] = CMessageQueue::GetInstance(GENERATE_KEY(shmQueueIden, CurType), shmQueueSize, eQueueModel::ONE_READ_ONE_WRITE); + } + if (serverEp->lastCopyLen_[CurType] != 0 && serverEp->lastCopyType_ == CurType) { // avoid pop the empty queue + serverEp->shmQueues_[CurType - 1]->Pop(serverEp->lastCopyLen_[CurType]); + } + uint8_t* src = NULL; + size_t len = 0; + int error_code = serverEp->shmQueues_[CurType - 1]->ReadMessage(&src, &len); //todo + tool::Logging(myName,"ZWX: kind=%d CurType=%02x error_code=%d len=%zu count=%zu last_copyLen=%zu dst=%p\n", kind, CurType, error_code, len, count, serverEp->lastCopyLen_[CurType], dst); + while(error_code != 0){ // blocking read until data is ready in shared memory queue + // tool::Logging(myName, "HYF# repeated reading"); + // tool::Logging(myName, "read failed from shared memory queue: %zu\n", len); + error_code = serverEp->shmQueues_[CurType - 1]->ReadMessage(&src, &len); + } + if (len == count) { + tool::Logging(myName, "cudaMemcpyAsync H2D: read data (%zu B) from shm queue(type=%02x)\n", len, CurType); + // printf("data[0-7]: "); + // for (size_t i = 0; i < 8; i++) { + // printf("%02x ", src[i]); + // } + // printf(", data[%zu-%zu]: ", len - 8, len - 1); + // for (size_t i = len - 8; i < len; i++) { + // printf("%02x ", src[i]); + // } + // printf("\n"); + + cudaError_t exit_code = cudaMemcpyAsync(dst, src, len, kind, stream); + if (exit_code == cudaSuccess) { + //cudaStreamSynchronize(stream); // make sure the data has been copied completely + serverEp->lastCopyLen_[CurType] = len; + serverEp->lastCopyType_ = CurType; + tool::Logging(myName, "cudaMemcpyAsync H2D: Host(%p) -> Device(%p) with %zu B\n", src, dst, count); +#ifdef GV_Scheduler + ++serverEp->numIterations; + if((serverEp->numIterations & 1) == 0) { + serverEp->sche->cal_add_It(serverEp->clientID_); + } +#endif + // return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync H2D failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync H2D failed: read failed from shm queue, len(%zu B) != cnt(%zu B)\n", len, count); + return UCS_ERR_IO_ERROR; + } + } + else { // without data preloading, data transfer from remote host to current device + if (count != length) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync H2D failed: invalid count(%zu), recv length(%zu)\n", count, length); + return UCS_ERR_IO_ERROR; + } + + if (param->recv_attr & UCP_AM_RECV_ATTR_FLAG_RNDV) { + tool::Logging(myName, "cudaMemcpyAsync H2D: RNDV request.\n"); + Request_t recv_request_ctx = {.type = 1, .complete = 0}; + ucp_request_param_t param2; + param2.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | + UCP_OP_ATTR_FIELD_DATATYPE | + UCP_OP_ATTR_FIELD_USER_DATA| + UCP_OP_ATTR_FIELD_MEMORY_TYPE; + param2.op_attr_mask |= UCP_OP_ATTR_FLAG_NO_IMM_CMPL; + param2.datatype = ucp_dt_make_contig(1); + param2.user_data = &recv_request_ctx; + param2.cb.recv_am = (ucp_am_recv_data_nbx_callback_t)RecvCallBack; + param2.memory_type = UCS_MEMORY_TYPE_CUDA; + Request_t* rndv_request = (Request_t*)ucp_am_recv_data_nbx(serverEp->_dataWorker, data, dst, count, ¶m2); + status = Wait(rndv_request, &recv_request_ctx, &serverEp->_dataWorker); + ucp_request_free(rndv_request); + if (status == UCS_OK) { + tool::Logging(myName, "cudaMemcpyAsync H2D: Host -> Device(%p) with %zu B\n", dst, count); + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync H2D failed: ucp_am_recv_data_nbx(%s)\n", ucs_status_string(status)); + return status; + } + // return status; + } + else { + tool::Logging(myName, "cudaMemcpyH2D: Eager request.\n"); + cudaError_t exit_code = cudaMemcpyAsync(dst, data, count, kind, stream); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemcpyAsync H2D: Host -> Device(%p) with %zu B\n", dst, count); + // return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyH2D failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } + } + } + return UCS_OK; +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyAsyncD2HHandle) { + tool::Logging(myName, "CUDA_MEMCPY_ASYNC_D2H\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaMemcpyKind kind = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + void* src = serverEp->GetDevPtr(reqBuf.Pop()); + + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_MEMCPY_ASYNC_D2H); + resBuf.Push((uint8_t*)src, count); + serverEp->SendResponse(&resBuf, ¶m->reply_ep, UCS_MEMORY_TYPE_CUDA); + + tool::Logging(myName, "cudaMemcpyAsync D2H: Device(%p) -> Host with %zu B\n", src, count); + return UCS_OK; +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyAsyncD2DHandle) { + tool::Logging(myName, "CUDA_MEMCPY_ASYNC_D2D\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaMemcpyKind kind = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + void* src = serverEp->GetDevPtr(reqBuf.Pop()); + void* dst = serverEp->GetDevPtr(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + cudaError_t exit_code = cudaMemcpyAsync(dst, src, count, kind, stream); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemcpyAsync D2D: Device(%p) -> Device(%p) with %zu B\n", src, dst, count); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync D2D failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaMemcpyToSymbolHandle) { + tool::Logging(myName, "CUDA_MEMCPY_TO_SYMBOL\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaMemcpyKind kind = reqBuf.Pop(); + size_t count = reqBuf.Pop(); + size_t offset = reqBuf.Pop(); + + if (kind == cudaMemcpyHostToDevice || kind == cudaMemcpyDefault) { + uint64_t symbol = reqBuf.Pop(); + void* src = reqBuf.AssignAddrForAll(); + + + auto it = serverEp->_cuInfoMap.mapHostVar2CuDevPtr->find(symbol); + if (it == serverEp->_cuInfoMap.mapHostVar2CuDevPtr->end()) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyToSymbolHandle failed: unknown symbol(%p)\n", symbol); + return UCS_ERR_IO_ERROR; + } + void* cuDevPtr = (void*)it->second; + // cudaError_t exit_code = cudaMemcpyToSymbol((void*)cuDevPtr, src, count, offset, kind); + cudaError_t exit_code = cudaMemcpy((uint8_t*)cuDevPtr + offset, src, count, kind); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaMemcpyToSymbol success\n"); + return UCS_OK; + } + else { + tool::Logging(myName, "cudaMemcpyToSymbol failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } + } + else { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyToSymbol failed: Unknown cudaMemcpyKind: %d\n", kind); + return UCS_ERR_IO_ERROR; + } +} + + diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtOtherCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtOtherCallback.cc new file mode 100644 index 0000000..c2a8ef0 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtOtherCallback.cc @@ -0,0 +1,80 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "CUDARuntimeOtherHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudaOccupancyMaxActiveBlocksPerMultiprocessorHandle) { + tool::Logging(myName, "CUDA_OCCUPANCY_MAX_ACTIVE_BLOCKS_PER_MULTIPROCESSOR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t hostFun = reqBuf.Pop(); + int blockSize = reqBuf.Pop(); + size_t dynSMemSize = reqBuf.Pop(); + + CUfunction cuFunc = NULL; + auto it = serverEp->_cuInfoMap.mapHost2CuFunc->find(hostFun); + if (it == serverEp->_cuInfoMap.mapHost2CuFunc->end()) { + tool::Logging(LOG_ERROR, myName, "cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags failed: unknown function(%p)\n", hostFun); + return UCS_ERR_IO_ERROR; + } + else { + cuFunc = it->second; + } + + cudaSetDevice(serverEp->curDev_); + + int numBlocks = 0; + CUresult exit_code = cuOccupancyMaxActiveBlocksPerMultiprocessor(&numBlocks, cuFunc, blockSize, dynSMemSize); + if (exit_code == CUDA_SUCCESS) { + tool::Logging(myName, "cudaOccupancyMaxActiveBlocksPerMultiprocessor success, numBlocks = %d\n", numBlocks); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_OCCUPANCY_MAX_ACTIVE_BLOCKS_PER_MULTIPROCESSOR); + resBuf.Push(numBlocks); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "cudaOccupancyMaxActiveBlocksPerMultiprocessor failed: %s\n", errorStr); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlagsHandle) { + tool::Logging(myName, "CUDA_OCCUPANCY_MAX_ACTIVE_BLOCKS_PER_MULTIPROCESSOR_WITH_FLAGS\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t hostFun = reqBuf.Pop(); + int blockSize = reqBuf.Pop(); + size_t dynSMemSize = reqBuf.Pop(); + unsigned int flags = reqBuf.Pop(); + + CUfunction cuFunc = NULL; + auto it = serverEp->_cuInfoMap.mapHost2CuFunc->find(hostFun); + if (it == serverEp->_cuInfoMap.mapHost2CuFunc->end()) { + tool::Logging(LOG_ERROR, myName, "cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags failed: unknown function(%p)\n", hostFun); + return UCS_ERR_IO_ERROR; + } + else { + cuFunc = it->second; + } + + cudaSetDevice(serverEp->curDev_); + + int numBlocks = 0; + CUresult exit_code = cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(&numBlocks, cuFunc, blockSize, dynSMemSize, flags); + if (exit_code == CUDA_SUCCESS) { + tool::Logging(myName, "cudaOccupancyMaxActiveBlocksPerMultiprocessor with Flags success, numBlocks = %d\n", numBlocks); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_OCCUPANCY_MAX_ACTIVE_BLOCKS_PER_MULTIPROCESSOR_WITH_FLAGS); + resBuf.Push(numBlocks); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + const char* errorStr; + cuGetErrorString(exit_code, &errorStr); + tool::Logging(LOG_ERROR, myName, "cudaOccupancyMaxActiveBlocksPerMultiprocessor with Flags failed: %s\n", errorStr); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtStreamCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtStreamCallback.cc new file mode 100644 index 0000000..a49f890 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/cudaRtStreamCallback.cc @@ -0,0 +1,312 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "CUDARuntimeStreamHandle"; + +DEFINE_SERVER_AM_CALLBACK(cudaStreamCreateHandle) { + tool::Logging(myName, "CUDA_STREAM_CREATE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudaStream_t stream; + cudaError_t exit_code = cudaStreamCreate(&stream); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamCreate success, pStream ptr: %p\n", stream); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, stream); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_STREAM_CREATE); + stream = (cudaStream_t)serverEp->GetHandleVirAddr(stream, CUDA_STREAM_CREATE); + resBuf.Push64BitPointer(stream); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamCreate failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + + +DEFINE_SERVER_AM_CALLBACK(cudaStreamDestroyHandle) { + tool::Logging(myName, "CUDA_STREAM_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop(), true); + cudaError_t exit_code = cudaStreamDestroy(stream); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamDestroy success, Stream ptr: %p\n", stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamDestroy failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + + +DEFINE_SERVER_AM_CALLBACK(cudaStreamCreateWithFlagsHandle) { + tool::Logging(myName, "CUDA_STREAM_CREATE_WITH_FLAGS\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudaStream_t stream; + unsigned int flags = reqBuf.Pop(); + cudaError_t exit_code = cudaStreamCreateWithFlags(&stream, flags); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamCreateWithFlags success, Stream ptr: %p, flags=%u\n", stream, flags); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, stream); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_STREAM_CREATE_WITH_FLAGS); + stream = (cudaStream_t)serverEp->GetHandleVirAddr(stream, CUDA_STREAM_CREATE_WITH_FLAGS); + resBuf.Push64BitPointer(stream); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamCreateWithFlags failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} +// end + +DEFINE_SERVER_AM_CALLBACK(cudaStreamCreateWithPriorityHandle) { + tool::Logging(myName, "CUDA_STREAM_CREATE_WITH_PRIORITY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + cudaStream_t stream; + unsigned int flags = reqBuf.Pop(); + int priority = reqBuf.Pop(); + cudaError_t exit_code = cudaStreamCreateWithPriority(&stream, flags, priority); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamCreateWithPriority success, Stream ptr: %p, flags = %u, priority = %d\n", stream, flags, priority); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, stream); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_STREAM_CREATE_WITH_PRIORITY); + stream = (cudaStream_t)serverEp->GetHandleVirAddr(stream, CUDA_STREAM_CREATE_WITH_PRIORITY); + resBuf.Push64BitPointer(stream); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamCreateWithPriority failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaStreamWaitEventHandle) { + tool::Logging(myName, "CUDA_STREAM_WAIT_EVENT\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + uint64_t eventVirAddr= reqBuf.Pop(); + cudaEvent_t event = (cudaEvent_t)serverEp->GetHandle(eventVirAddr); + uint flags = reqBuf.Pop(); + if (event == NULL) { + tool::Logging(LOG_ERROR, myName, "cudaStreamWaitEvent warning: eventPtr is NULL(virAddr=%p), streamPtr=%p\n", eventVirAddr, stream); + return UCS_OK; + } + + cudaError_t exit_code = cudaStreamWaitEvent(stream, event, flags); + if (exit_code == cudaSuccess) { + bool isCommEvent = false; +#ifdef GV_BACKUP + Handle_t* handle = serverEp->GetHandleInfo(eventVirAddr); + isCommEvent = (serverEp->commStream_) && (serverEp->commStream_ == (cudaStream_t)handle->stream); + if (isCommEvent) { + boost::unique_lock lock(serverEp->eventWatchedSync_.mutex); + serverEp->watchedEventsList_.push_back(event); + } +#endif // GV_BACKUP + tool::Logging(myName, "cudaStreamWaitEvent success, eventPtr=%p(isCommEvent=%d), streamPtr=%p, flags = %u\n", event, isCommEvent, stream, flags); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamWaitEvent failed: %s, eventPtr=%p(virAddr=%p), streamPtr=%p\n", cudaGetErrorName(exit_code), event, eventVirAddr, stream); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaStreamSynchronizeHandle) { + tool::Logging(myName, "CUDA_STREAM_SYNCHRONIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + +#ifdef GV_BACKUP + // debug: + // if (serverEp->ckptCnt >= 2) { + // // if (serverEp->curIter_ >= BACKUP_PERIOD + 1) { + // // if (serverEp->curIter_ >= BACKUP_PERIOD + BACKUP_PERIOD - 5) { + // tool::Logging(LOG_INFO, myName, "Ready to close the server endpoint object.\n"); + // // { + // // boost::unique_lock lock(serverEp->backupSync_.mutex); + // // serverEp->backupSync_.cv.wait(lock, [serverEp] { return !serverEp->bufferReady_; }); + // // } + // serverEp->stateBackup_.Wait(); + + // for (const auto stream : serverEp->streamList_) { + // cudaStreamSynchronize(stream); + // } + + // /* + // size_t totalTensorsSize = 0; + // for (const auto& tensor : serverEp->trainTensors_) { + // totalTensorsSize += tensor.size; + // } + // totalTensorsSize = totalTensorsSize * BACKUP_PERIOD; + // std::string fileName = serverEp->backupFilePath_ + ".tmp"; + // std::ofstream ofs(fileName.c_str(), std::ios::binary | std::ios::out | std::ios::trunc); + // if (!ofs) { + // throw std::runtime_error("Opening file failed: " + fileName); + // } + // ofs.seekp(totalTensorsSize - 1); + // ofs.write("", 1); + // ofs.close(); + // boost::interprocess::file_mapping fileMapping(fileName.c_str(), boost::interprocess::read_write); + // boost::interprocess::mapped_region mappingRegion(fileMapping, boost::interprocess::read_write, 0, totalTensorsSize); + // auto start = std::chrono::high_resolution_clock::now(); + // std::memcpy(mappingRegion.get_address(), serverEp->backupMemcpyBuffer_, totalTensorsSize); + // auto end = std::chrono::high_resolution_clock::now(); + // tool::Logging(LOG_INFO, myName, "training data is written to file(%s) with size %zu bytes in %f seconds\n", fileName.c_str(), totalTensorsSize, std::chrono::duration(end - start).count()); + // */ + + // // exit(EXIT_FAILURE); + + // serverEp->connStatus_.isClosed = true; + // for (int i = 0; i < 3; i++) serverEp->shmQueSizes_[i]->SaveState(); + // return UCS_ERR_CONNECTION_RESET; + // } +#ifdef GV_MSGHANDLER + // if (serverEp->MSGStopFlag_) { + if (serverEp->MSGStopFlag_ && !serverEp->msgWaitFlag_){ + tool::Logging(LOG_INFO, myName, "Ready to close the server endpoint object.\n"); + serverEp->msgResetCnt = serverEp->ckptCnt; + serverEp->msgWaitFlag_ = true; + } + else if (serverEp->MSGStopFlag_ && serverEp->msgWaitFlag_ && (serverEp->ckptCnt >= serverEp->msgResetCnt + 1)) { + // if (serverEp->curIter_ >= BACKUP_PERIOD + 1) { + // if (serverEp->curIter_ >= BACKUP_PERIOD + BACKUP_PERIOD - 5) { + tool::Logging(LOG_INFO, myName, "Ready to close the server endpoint object.\n"); + // { + // boost::unique_lock lock(serverEp->backupSync_.mutex); + // serverEp->backupSync_.cv.wait(lock, [serverEp] { return !serverEp->bufferReady_; }); + // } + serverEp->stateBackup_.Wait(); + + for (const auto stream : serverEp->streamList_) { + cudaStreamSynchronize(stream); + } + + /* + size_t totalTensorsSize = 0; + for (const auto& tensor : serverEp->trainTensors_) { + totalTensorsSize += tensor.size; + } + totalTensorsSize = totalTensorsSize * BACKUP_PERIOD; + std::string fileName = serverEp->backupFilePath_ + ".tmp"; + std::ofstream ofs(fileName.c_str(), std::ios::binary | std::ios::out | std::ios::trunc); + if (!ofs) { + throw std::runtime_error("Opening file failed: " + fileName); + } + ofs.seekp(totalTensorsSize - 1); + ofs.write("", 1); + ofs.close(); + boost::interprocess::file_mapping fileMapping(fileName.c_str(), boost::interprocess::read_write); + boost::interprocess::mapped_region mappingRegion(fileMapping, boost::interprocess::read_write, 0, totalTensorsSize); + auto start = std::chrono::high_resolution_clock::now(); + std::memcpy(mappingRegion.get_address(), serverEp->backupMemcpyBuffer_, totalTensorsSize); + auto end = std::chrono::high_resolution_clock::now(); + tool::Logging(LOG_INFO, myName, "training data is written to file(%s) with size %zu bytes in %f seconds\n", fileName.c_str(), totalTensorsSize, std::chrono::duration(end - start).count()); + */ + + // exit(EXIT_FAILURE); + + serverEp->connStatus_.isClosed = true; + for (int i = 0; i < 3; i++) serverEp->shmQueSizes_[i]->SaveState(); + return UCS_ERR_CONNECTION_RESET; + } + + #endif // GV_MSGHANDLER +#endif // GV_BACKUP + + cudaError_t exit_code = cudaStreamSynchronize(stream); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_STREAM_SYNCHRONIZE); + resBuf.Push(exit_code); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamSynchronize success, pStream=%p\n",stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamSynchronize failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(cudaStreamIsCapturingHandle) { + tool::Logging(myName, "CUDA_STREAM_IS_CAPTURING\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + enum cudaStreamCaptureStatus pCaptureStatus; + cudaError_t exit_code = cudaStreamIsCapturing(stream, &pCaptureStatus); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamIsCapturing success, pStream=%p, pCaptureStatus=%d\n",stream, pCaptureStatus); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_STREAM_IS_CAPTURING); + resBuf.Push(pCaptureStatus); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamIsCapturing failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} +// end + + +DEFINE_SERVER_AM_CALLBACK(cudaStreamGetCaptureInfoHandle) { + tool::Logging(myName, "CUDA_STREAM_GET_CAPTURE_INFO\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + // debug: 这里pop出来的是pStream, 而非stream + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + stream = (stream == NULL) ? serverEp->defaultStream_ : stream; + enum cudaStreamCaptureStatus pCaptureStatus; + unsigned long long pId; + cudaError_t exit_code = cudaStreamGetCaptureInfo(stream, &pCaptureStatus, &pId); + if (exit_code == cudaSuccess) { + tool::Logging(myName, "cudaStreamGetCaptureInfo success, streamPtr=%p, pCaptureStatus=%d, pId=%llu\n",stream, pCaptureStatus, pId); + // send pCaptureStatus and pId of the stream back + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(CUDA_STREAM_GET_CAPTURE_INFO); + resBuf.Push(pCaptureStatus); + resBuf.Push(pId); //todo: handle mapping + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "cudaStreamGetCaptureInfo failed: %s\n", cudaGetErrorName(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + + + diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/ncclCallback.cc b/GPU-Virtual-Service/gpu-remoting/src/server/ncclCallback.cc new file mode 100644 index 0000000..1f250ec --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/ncclCallback.cc @@ -0,0 +1,689 @@ +#include "../../include/serverEndpoint.h" + +static const char* myName = "ncclHandle"; + +DEFINE_SERVER_AM_CALLBACK(ncclMemAllocHandle) { + tool::Logging(myName, "NCCL_MEM_ALLOC\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + void* devPtr = NULL; + size_t cuda_size = reqBuf.Pop(); + ncclResult_t exit_code = ncclMemAlloc(&devPtr, cuda_size); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclMemAlloc success, devPtr = %p with size = %zu\n", devPtr, cuda_size); + uint64_t virtAddr = (uint64_t)devPtr; + +#ifdef GV_MEMORY + virtAddr = serverEp->_cuInfoMap.blockManager->AddBlock((uint64_t)devPtr, cuda_size, true); +#else + size_t blockNum = ( + serverEp->_cuInfoMap.blockInfoList.emplace_back(Block_t{.devPtr = (uint64_t)devPtr, .size = cuda_size, .valid = true}), + serverEp->_cuInfoMap.blockInfoList.size() - 1 + ); +#endif // GV_MEMORY + +#ifdef GV_MEMORY_PTX + if (blockNum >= BLOCKS_MAX_NUM) { + tool::Logging(LOG_ERROR, myName, "ncclMemAlloc failed: current blockNum(%zu) >= BLOCKS_MAX_NUM(%zu)\n", blockNum, BLOCKS_MAX_NUM); + return UCS_ERR_IO_ERROR; + } + virtAddr = blockNum << BLOCK_SHIFT_BIT; +#endif // GV_MEMORY_PTX + + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_MEM_ALLOC); + // resBuf.Push64BitPointer(devPtr); + resBuf.Push(virtAddr); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclMemAlloc failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclMemFreeHandle) { + tool::Logging(myName, "NCCL_MEM_FREE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virtAddr = reqBuf.Pop(); + void* devPtr = serverEp->GetDevPtr(virtAddr); + ncclResult_t exit_code = ncclMemFree(devPtr); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclMemFree success, devPtr: %p\n", devPtr); +#ifdef GV_MEMORY_PTX + size_t blockIdx = serverEp->GetBlockIdx(virtAddr); + serverEp->_cuInfoMap.blockInfoList[blockIdx].valid = false; +#endif // GV_MEMORY + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclMemFree failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclGroupStartHandle) { + tool::Logging(myName, "NCCL_GROUP_START\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclResult_t exit_code = ncclGroupStart(); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclGroupStart success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclGroupStart failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclGroupEndHandle) { + tool::Logging(myName, "NCCL_GROUP_END\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclResult_t exit_code = ncclGroupEnd(); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclGroupEnd success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclGroupEnd failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommInitRankHandle) { + tool::Logging(myName, "NCCL_COMM_INIT_RANK\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + int nranks = reqBuf.Pop(); + ncclUniqueId commId = reqBuf.Pop(); + int rank = reqBuf.Pop(); + ncclComm_t comm = NULL; + ncclResult_t exit_code = ncclCommInitRank(&comm, nranks, commId, rank); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommInitRank success, comm: %p, nranks: %d, rank: %d\n", comm, nranks, rank); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, comm); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_INIT_RANK); + comm = (ncclComm_t)serverEp->GetHandleVirAddr(comm, NCCL_COMM_INIT_RANK); + resBuf.Push64BitPointer(comm); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + serverEp->curComm = comm; + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommInitRank failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommInitAllHandle) { + tool::Logging(myName, "NCCL_COMM_INIT_ALL\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t* virAddr = reqBuf.AssignAddr(); + int ndev = reqBuf.Pop(); + const int* devlist = reqBuf.AssignAddr(); + ncclComm_t comms[ndev]; // todo: may be invalid + ncclResult_t exit_code = ncclCommInitAll(comms, ndev, devlist); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommInitAll success, ndev: %d\n", ndev); + if (virAddr[0] != 0) { + for (int i = 0; i < ndev; i++) { + serverEp->SetHandleVirAddr(virAddr[i], comms[i]); + } + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_INIT_ALL); + for (int i = 0; i < ndev; i++) { + comms[i] = (ncclComm_t)serverEp->GetHandleVirAddr(comms[i], NCCL_COMM_INIT_ALL); + + } + resBuf.Push((uint64_t*)comms, ndev); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommInitAll failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommInitRankConfigHandle) { + tool::Logging(myName, "NCCL_COMM_INIT_RANK_CONFIG\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + uint64_t virAddr = reqBuf.Pop(); + int nranks = reqBuf.Pop(); + ncclUniqueId commId = reqBuf.Pop(); + int rank = reqBuf.Pop(); + ncclConfig_t* config = reqBuf.AssignAddr(); + char* netName = reqBuf.AssignAddr(); + config->netName = netName; + ncclComm_t comm = NULL; + ncclResult_t exit_code = ncclCommInitRankConfig(&comm, nranks, commId, rank, config); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommInitRankConfig success, comm: %p, nranks: %d, rank: %d\n", comm, nranks, rank); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, comm); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_INIT_RANK_CONFIG); + comm = (ncclComm_t)serverEp->GetHandleVirAddr(comm, NCCL_COMM_INIT_RANK_CONFIG); + resBuf.Push64BitPointer(comm); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + serverEp->curComm = comm; + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommInitRankConfig failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommDestroyHandle) { + tool::Logging(myName, "NCCL_COMM_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop(), true); + ncclResult_t exit_code = ncclCommDestroy(comm); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommDestroy success, comm: %p\n", comm); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommDestroy failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommSplitHandle) { + tool::Logging(myName, "NCCL_COMM_SPLIT\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + int color = reqBuf.Pop(); + int key = reqBuf.Pop(); + uint64_t virAddr = reqBuf.Pop(); + ncclConfig_t* config = reqBuf.AssignAddr(); + char* netName = reqBuf.AssignAddr(); + config->netName = netName; + ncclComm_t newcomm = NULL; + ncclResult_t exit_code = ncclCommSplit(comm, color, key, &newcomm, config); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommSplit success, oldcomm: %p, newcomm: %p\n", comm, newcomm); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, newcomm); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_SPLIT); + newcomm = (ncclComm_t)serverEp->GetHandleVirAddr(newcomm, NCCL_COMM_SPLIT); + resBuf.Push64BitPointer(newcomm); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommSplit failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommFinalizeHandle) { + tool::Logging(myName, "NCCL_COMM_FINALIZE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + ncclResult_t exit_code = ncclCommFinalize(comm); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommFinalize success, comm: %p\n", comm); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommFinalize failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommGetAsyncErrorHandle) { + tool::Logging(myName, "NCCL_COMM_GET_ASYNC_ERROR\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + ncclResult_t asyncError = ncclSuccess; + ncclResult_t exit_code = ncclCommGetAsyncError(comm, &asyncError); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommGetAsyncError success, asyncError: %d\n", asyncError); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_GET_ASYNC_ERROR); + resBuf.Push(asyncError); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommGetAsyncError failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommCountHandle) { + tool::Logging(myName, "NCCL_COMM_COUNT\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + int count = 0; + ncclResult_t exit_code = ncclCommCount(comm, &count); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommCount success, count: %d\n", count); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_COUNT); + resBuf.Push(count); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommCount failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommUserRankHandle) { + tool::Logging(myName, "NCCL_COMM_USER_RANK\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + int rank = 0; + ncclResult_t exit_code = ncclCommUserRank(comm, &rank); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommUserRank success, rank: %d\n", rank); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_USER_RANK); + resBuf.Push(rank); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommUserRank failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommCuDeviceHandle) { + tool::Logging(myName, "NCCL_COMM_CU_DEVICE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + int device = 0; + ncclResult_t exit_code = ncclCommCuDevice(comm, &device); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommCuDevice success, device: %d\n", device); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_CU_DEVICE); + resBuf.Push(device); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommCuDevice failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommAbortHandle) { + tool::Logging(myName, "NCCL_COMM_ABORT\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + ncclResult_t exit_code = ncclCommAbort(comm); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommAbort success\n"); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommAbort failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommRegisterHandle) { + tool::Logging(myName, "NCCL_COMM_REGISTER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const ncclComm_t comm = (const ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + void* buff = (void*)reqBuf.Pop(); + size_t size = reqBuf.Pop(); + uint64_t virAddr = reqBuf.Pop(); + void* handle = NULL; + buff = serverEp->GetDevPtr((uint64_t)buff); + ncclResult_t exit_code = ncclCommRegister(comm, buff, size, &handle); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommRegister success, buff: %p, handle: %p\n", buff, handle); + if (virAddr != 0) { + serverEp->SetHandleVirAddr(virAddr, handle); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_COMM_REGISTER); + handle = serverEp->GetHandleVirAddr(handle, NCCL_COMM_REGISTER); + resBuf.Push64BitPointer(handle); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommRegister failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclCommDeregisterHandle) { + tool::Logging(myName, "NCCL_COMM_DEREGISTER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const ncclComm_t comm = (const ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + void* handle = (void*)serverEp->GetHandle(reqBuf.Pop()); + ncclResult_t exit_code = ncclCommDeregister(comm, handle); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclCommDeregister success, handle: %p\n", handle); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclCommDeregister failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclGetUniqueIdHandle) { + tool::Logging(myName, "NCCL_GET_UNIQUE_ID\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclUniqueId uniqueId; + ncclResult_t exit_code = ncclGetUniqueId(&uniqueId); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclGetUniqueId success\n"); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_GET_UNIQUE_ID); + resBuf.Push(uniqueId); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclGetUniqueId failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclGetVersionHandle) { + tool::Logging(myName, "NCCL_GET_VERSION\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + int version; + ncclResult_t exit_code = ncclGetVersion(&version); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclGetVersion success, version: %d\n", version); + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_GET_VERSION); + resBuf.Push(version); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclGetVersion failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } + +} + +DEFINE_SERVER_AM_CALLBACK(ncclAllReduceHandle) { + tool::Logging(myName, "NCCL_ALL_REDUCE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const void* sendbuff = (const void*) reqBuf.Pop(); + void* recvbuff = (void*) reqBuf.Pop(); + size_t count = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + ncclRedOp_t op = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + sendbuff = serverEp->GetDevPtr((uint64_t)sendbuff); + recvbuff = serverEp->GetDevPtr((uint64_t)recvbuff); + + // // debug: + // if (serverEp->ckptCnt >= 2) { + // ncclCommAbort(comm); + // ncclCommDestroy(comm); + // { + // boost::unique_lock lock(serverEp->backupSync_.mutex); + // serverEp->backupSync_.cv.wait(lock, [serverEp] { return !serverEp->bufferReady_; }); + // } + // sleep(3); // wait for the client to finish shrinking of api list + // exit(EXIT_FAILURE); + // // serverEp->connStatus_.isClosed = true; + // // for (int i = 0; i < 3; i++) serverEp->shmQueSizes_[i]->SaveState(); + // // return UCS_ERR_CONNECTION_RESET; + // } + + ncclResult_t exit_code = ncclAllReduce(sendbuff, recvbuff, count, datatype, op, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclAllReduce success, sendbuff: %p, recvbuff: %p, count: %d, stream: %p\n", sendbuff, recvbuff, count, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclAllReduce failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclReduceHandle) { + tool::Logging(myName, "NCCL_REDUCE\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const void* sendbuff = (const void*) reqBuf.Pop(); + void* recvbuff = (void*) reqBuf.Pop(); + size_t count = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + ncclRedOp_t op = reqBuf.Pop(); + int root = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + sendbuff = serverEp->GetDevPtr((uint64_t)sendbuff); + recvbuff = serverEp->GetDevPtr((uint64_t)recvbuff); + ncclResult_t exit_code = ncclReduce(sendbuff, recvbuff, count, datatype, op, root, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclReduce success, sendbuff: %p, recvbuff: %p, count: %d, stream: %p\n", sendbuff, recvbuff, count, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclReduce failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclReduceScatterHandle) { + tool::Logging(myName, "NCCL_REDUCE_SCATTER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const void* sendbuff = (const void*) reqBuf.Pop(); + void* recvbuff = (void*) reqBuf.Pop(); + size_t recvcount = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + ncclRedOp_t op = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + sendbuff = serverEp->GetDevPtr((uint64_t)sendbuff); + recvbuff = serverEp->GetDevPtr((uint64_t)recvbuff); + ncclResult_t exit_code = ncclReduceScatter(sendbuff, recvbuff, recvcount, datatype, op, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclReduceScatter success, sendbuff: %p, recvbuff: %p, recvcount: %d, stream: %p\n", sendbuff, recvbuff, recvcount, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclReduceScatter failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclAllGatherHandle) { + tool::Logging(myName, "NCCL_ALL_GATHER\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const void* sendbuff = (const void*) reqBuf.Pop(); + void* recvbuff = (void*) reqBuf.Pop(); + size_t sendcount = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + sendbuff = serverEp->GetDevPtr((uint64_t)sendbuff); + recvbuff = serverEp->GetDevPtr((uint64_t)recvbuff); + ncclResult_t exit_code = ncclAllGather(sendbuff, recvbuff, sendcount, datatype, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclAllGather success, sendbuff: %p, recvbuff: %p, sendcount: %d, stream: %p\n", sendbuff, recvbuff, sendcount, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclAllGather failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclBroadcastHandle) { + tool::Logging(myName, "NCCL_BROADCAST\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const void* sendbuff = (const void*) reqBuf.Pop(); + void* recvbuff = (void*) reqBuf.Pop(); + size_t count = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + int root = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + sendbuff = serverEp->GetDevPtr((uint64_t)sendbuff); + recvbuff = serverEp->GetDevPtr((uint64_t)recvbuff); + ncclResult_t exit_code = ncclBroadcast(sendbuff, recvbuff, count, datatype, root, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclBroadcast success, sendbuff: %p, recvbuff: %p, count: %d, stream: %p\n", sendbuff, recvbuff, count, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclBroadcast failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclSendHandle) { + tool::Logging(myName, "NCCL_SEND\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + const void* sendbuff = (const void*) reqBuf.Pop(); + size_t count = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + int peer = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + sendbuff = serverEp->GetDevPtr((uint64_t)sendbuff); + ncclResult_t exit_code = ncclSend(sendbuff, count, datatype, peer, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclSend success, sendbuff: %p, count: %d, stream: %p\n", sendbuff, count, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclSend failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclRecvHandle) { + tool::Logging(myName, "NCCL_RECV\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + void* recvbuff = (void*) reqBuf.Pop(); + size_t count = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + int peer = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + cudaStream_t stream = (cudaStream_t)serverEp->GetHandle(reqBuf.Pop()); + recvbuff = serverEp->GetDevPtr((uint64_t)recvbuff); + ncclResult_t exit_code = ncclRecv(recvbuff, count, datatype, peer, comm, stream); + if (exit_code == ncclSuccess) { + serverEp->UpdateStream(stream); + tool::Logging(myName, "ncclRecv success, recvbuff: %p, count: %d, stream: %p\n", recvbuff, count, stream); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclRecv failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclRedOpCreatePreMulSumHandle) { + tool::Logging(myName, "NCCL_RED_OP_CREATE_PRE_MUL_SUM\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclRedOp_t virOp = reqBuf.Pop(); + ncclDataType_t datatype = reqBuf.Pop(); + ncclScalarResidence_t residence = reqBuf.Pop(); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + void* scalar = NULL; + ncclRedOp_t op; + if (residence == ncclScalarHostImmediate) { + scalar = reqBuf.AssignAddr(); + } else { + scalar = (void*)reqBuf.Pop(); + scalar = serverEp->GetDevPtr((uint64_t)scalar); + } + ncclResult_t exit_code = ncclRedOpCreatePreMulSum(&op, scalar, datatype, residence, comm); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclRedOpCreatePreMulSum success, scalar: %p, op: %d\n", scalar, op); + if (virOp != 0) { + serverEp->SetNcclRedOpVirAddr(virOp, op); + } + else { + RequestIOV resBuf = RequestIOV(reqBuf.GetThreadID()); + resBuf.PushRequestType(NCCL_RED_OP_CREATE_PRE_MUL_SUM); + op = serverEp->GetNcclRedOpVirAddr(op); + resBuf.Push(op); + serverEp->SendResponse(&resBuf, ¶m->reply_ep); + } + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclRedOpCreatePreMulSum failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} + +DEFINE_SERVER_AM_CALLBACK(ncclRedOpDestroyHandle) { + tool::Logging(myName, "NCCL_RED_OP_DESTROY\n"); + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + ncclRedOp_t op = serverEp->GetNcclRedOp(reqBuf.Pop(), true); + ncclComm_t comm = (ncclComm_t)serverEp->GetHandle(reqBuf.Pop()); + ncclResult_t exit_code = ncclRedOpDestroy(op, comm); + if (exit_code == ncclSuccess) { + tool::Logging(myName, "ncclRedOpDestroy success, op: %d\n", op); + return UCS_OK; + } + else { + tool::Logging(LOG_ERROR, myName, "ncclRedOpDestroy failed: %s\n", ncclGetErrorString(exit_code)); + return UCS_ERR_IO_ERROR; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/serverApp.cc b/GPU-Virtual-Service/gpu-remoting/src/server/serverApp.cc new file mode 100644 index 0000000..d8b7539 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/serverApp.cc @@ -0,0 +1,103 @@ +#include +#include "../../include/configure.h" +#include "../../include/ucpConnection.h" +#include "../../include/serverEndpoint.h" +#include "../../include/conqueue/readerwriterqueue.h" +using namespace std; + +Configure config_("config.json"); +const char* myName = "ServerApp"; +UCPConnection* connectionObj; +vector thList; +// moodycamel::BlockingReaderWriterQueue q(CONN_RESERVED_NUM); +volatile bool isClosed = false; + +void CTRLC(int s) { + tool::Logging(LOG_INFO, myName, "terminate the server with ctrl+c interrupt\n"); + __sync_add_and_fetch_8(&isClosed, 1); +} + +void cleanup() { + // for (auto it : thList) { + // it->join(); + // delete it; + // } + // for (auto it : serverEpList) { + // delete it; + // } + // tool::Logging(LOG_INFO, myName, "clear all server endpoint object.\n"); + + delete connectionObj; + tool::Logging(LOG_INFO, myName, "close the network connection.\n"); +} + +int main(int argc, char* argv[]) { + + struct sigaction sigIntHandler; + sigIntHandler.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &sigIntHandler, 0); // avoid the server crash when writing to a closed connection + sigIntHandler.sa_handler = CTRLC; + sigaction(SIGINT, &sigIntHandler, 0); + + // boost::thread* thTmp; + // boost::thread_attributes attrs; + // attrs.set_stack_size(THREAD_STACK_SIZE); + + int devCnt = 0; + cudaError_t exit_code = cudaGetDeviceCount(&devCnt); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaGetDeviceCount failed: %s\n", cudaGetErrorName(exit_code)); + exit(EXIT_FAILURE); + } + + + connectionObj = new UCPConnection(false); + connectionObj->Listen(config_.GetServerIp(), config_.GetServerPort(), ServerEndpoint::CreateServerEp); + connectionObj->WaitConnection(&isClosed); + + cleanup(); + + // while(true) { + // ucp_conn_request_h conn_request = NULL; + // uint64_t client_id = 0; + // char* clientIP = NULL; + // char* clientPort = NULL; + // connectionObj->WaitConnection(&conn_request, &client_id, &clientIP, &clientPort); + + // // ucp_worker_h dataWorker; + // // connectionObj->InitWorker(&dataWorker); + // // both dataWorker and conn_request are pointers to the struct + // serverEndPointObj = new ServerEndpoint(connectionObj->CreateWorker(), conn_request, client_id, clientIP, clientPort); + + // thTmp = new boost::thread(attrs, boost::bind(&ServerEndpoint::Run, serverEndPointObj)); + // thList.emplace_back(thTmp); + // //serverEpList.push_back(serverEndPointObj); + // } + + // boost::thread CreateServeEp([&]() { + // UCPConnection_t conn; + // while (!isClosed) { + // if (q.wait_dequeue_timed(conn, std::chrono::seconds(2))) { // wakeup for every 2 seconds + // serverEndPointObj = new ServerEndpoint(connectionObj->CreateWorker(), conn); + // thTmp = new boost::thread(attrs, boost::bind(&ServerEndpoint::NewRun, serverEndPointObj)); + // thList.emplace_back(thTmp); + // } + // } + // }); + + // boost::thread WaitConn([&]() { + // while (!isClosed) { + // UCPConnection_t conn; + // connectionObj->WaitConnection(&conn, &isClosed); + // if (isClosed) { + // break; + // } + // q.enqueue(conn); + // } + // }); + + // WaitConn.join(); + // CreateServeEp.join(); + + return 0; +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/serverBackup.cc b/GPU-Virtual-Service/gpu-remoting/src/server/serverBackup.cc new file mode 100644 index 0000000..278eb97 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/serverBackup.cc @@ -0,0 +1,657 @@ +#include "../../include/serverEndpoint.h" + +void ServerEndpoint::Backup2Memory() { + const char* myName = "Backup"; + tool::Logging(LOG_DEBUG, myName, "ready to backup the memory blocks and the handle mappings.\n"); + + auto start_total = std::chrono::high_resolution_clock::now(); + + // /* Wait for the sub thread to signal that the buffer is ready to be overwritten */ + // { + // boost::unique_lock lock(backupSync_.mutex); + // backupSync_.cv.wait(lock, [this] { return !bufferReady_; }); + // } + + stateBackup_.Wait(); + + /* prepare the backup buffer and prepare */ + auto start_prepare = std::chrono::high_resolution_clock::now(); + _cuInfoMap.handleManager->Shrink(); // remove the invalid handles + size_t totalDataBytes = 0, memcpyDataBytes = 0; + // const std::vector& handles = _cuInfoMap.handleManager->GetHandleInfoList(); + const std::vector>& handles = _cuInfoMap.handleManager->GetValidHandles(); + const std::vector& blocks = _cuInfoMap.blockManager->GetBlocks(); + + std::vector sortedTensors = trainTensors_; + std::sort(sortedTensors.begin(), sortedTensors.end(), + [](const TensorInfo_t& a, const TensorInfo_t& b) { + return a.devPtr < b.devPtr; + }); + std::vector> block2tensors(blocks.size()); + for (size_t i = 0; i < sortedTensors.size(); i++) { + uint64_t realAddr = 0; + int blockIdx = _cuInfoMap.blockManager->FindByVirAddr((uint64_t)sortedTensors[i].devPtr, realAddr); + if (blockIdx != -1) { + const Block_t& tmpBlock = blocks[blockIdx]; + sortedTensors[i].devPtr = (void*)realAddr; + if (tmpBlock.essential && realAddr + sortedTensors[i].size <= tmpBlock.devPtr + tmpBlock.size) { + block2tensors[blockIdx].push_back(&sortedTensors[i]); + tool::Logging(LOG_DEBUG, myName, "block[%d](addr=%p, size=%zu) has tensor[%zu](%p) with size %zu\n", blockIdx, (void*)tmpBlock.devPtr, tmpBlock.size, i, sortedTensors[i].devPtr, sortedTensors[i].size); + memcpyDataBytes += sortedTensors[i].size; + } + else if (tmpBlock.essential == false) { + tool::Logging(LOG_DEBUG, myName, "block[%d](%p) is not essential\n", blockIdx, (void*)tmpBlock.devPtr); + } + else { + tool::Logging(LOG_DEBUG, myName, "tensor[%zu](%p) is not in block[%d](%p)\n", i, sortedTensors[i].devPtr, blockIdx, (void*)tmpBlock.devPtr); + } + } + } + // size_t tmpOffset = 0; + for (size_t blockIdx = 0; blockIdx < blocks.size(); blockIdx++) { + if (blocks[blockIdx].valid == false || blocks[blockIdx].essential == false) { + continue; + } + // tool::Logging(LOG_INFO, myName, "offset=%zu, prepare to backup block[%zu]: devPtr=%p, size=%zu\n", tmpOffset, blockIdx, (void*)blocks[blockIdx].devPtr, blocks[blockIdx].size); + totalDataBytes += blocks[blockIdx].size; + // tmpOffset += blocks[blockIdx].size; + } + + size_t curBackupSize = sizeof(lastCopyLen_) + sizeof(lastCopyType_) + sizeof(size_t) * 3 + sizeof(handles[0]) * handles.size() + + sizeof(Block_t) * blocks.size() + sizeof(TensorInfo_t) * trainTensors_.size() + + (totalDataBytes - memcpyDataBytes); + // + totalDataBytes; + tool::Logging(LOG_DEBUG, myName, "prepare the backup buffer with %zu bytes (dataBytes: %zu, memcpyBytes: %zu)\n", curBackupSize, totalDataBytes, memcpyDataBytes); + if (serverState_.hostPtr == NULL || serverState_.size < curBackupSize) { + bufferResized_ = true; + serverState_.size = ALIGN_UP(curBackupSize); // align to 4KB + if (serverState_.hostPtr != NULL) { + // free(serverState_.hostPtr); + cudaFreeHost(serverState_.hostPtr); + } + cudaError_t exit_code = cudaMallocHost(&serverState_.hostPtr, serverState_.size); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMallocHost failed: %s\n", cudaGetErrorString(exit_code)); + exit(EXIT_FAILURE); + } + } + if (streamList_.size() == 0) { + streamList_.resize(BACKUP_STREAM_NUM, NULL); + for (int i = 0; i < streamList_.size(); i++) { + cudaStreamCreateWithFlags(&streamList_[i], cudaStreamNonBlocking); + } + } + auto end_prepare = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration_prepare = end_prepare - start_prepare; + tool::Logging(LOG_INFO, myName, "backup buffer is prepared with %zu bytes (isResided: %d) in %f seconds\n", serverState_.size, bufferResized_, duration_prepare.count()); + + + /* backup the mappings for the memory blocks and the handles */ + auto start_backup = std::chrono::high_resolution_clock::now(); + size_t bufferOffset = 0; + size_t handleNum = handles.size(); + size_t blockNum = blocks.size(); + size_t memcpyCnt = trainTensors_.size(); + memcpy(serverState_.hostPtr + bufferOffset, &lastCopyLen_, sizeof(lastCopyLen_)); + bufferOffset += sizeof(lastCopyLen_); + memcpy(serverState_.hostPtr + bufferOffset, &lastCopyType_, sizeof(lastCopyType_)); + bufferOffset += sizeof(lastCopyType_); + memcpy(serverState_.hostPtr + bufferOffset, &handleNum, sizeof(size_t)); + bufferOffset += sizeof(size_t); + memcpy(serverState_.hostPtr + bufferOffset, &blockNum, sizeof(size_t)); + bufferOffset += sizeof(size_t); + memcpy(serverState_.hostPtr + bufferOffset, &memcpyCnt, sizeof(size_t)); + bufferOffset += sizeof(size_t); + // memcpy(serverState_.hostPtr + bufferOffset, handles.data(), sizeof(Handle_t) * handleNum); + // bufferOffset += sizeof(Handle_t) * handleNum; + memcpy(serverState_.hostPtr + bufferOffset, handles.data(), sizeof(handles[0]) * handleNum); + bufferOffset += sizeof(handles[0]) * handleNum; + memcpy(serverState_.hostPtr + bufferOffset, blocks.data(), sizeof(Block_t) * blockNum); + bufferOffset += sizeof(Block_t) * blockNum; + memcpy(serverState_.hostPtr + bufferOffset, trainTensors_.data(), sizeof(TensorInfo_t) * trainTensors_.size()); + bufferOffset += sizeof(TensorInfo_t) * trainTensors_.size(); + auto end_backup = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration_backup = end_backup - start_backup; + tool::Logging(LOG_INFO, myName, "backup %zu (valid) handles and %zu blockInfos in %f seconds\n", handleNum, blockNum, duration_backup.count()); + + /* backup the memory blocks */ + auto start = std::chrono::high_resolution_clock::now(); + size_t essentialBlockNum = 0; + for (size_t blockIdx = 0; blockIdx < blocks.size(); blockIdx++) { + if (blocks[blockIdx].valid == false || blocks[blockIdx].essential == false) { + continue; + } + essentialBlockNum++; + + // printf("\n"); + tool::Logging(LOG_DEBUG, myName, "bufferOffset=%zu, prepare to backup block[%zu]: devPtr=%p, size=%zu\n", bufferOffset, blockIdx, (void*)blocks[blockIdx].devPtr, blocks[blockIdx].size); + + const Block_t& block = blocks[blockIdx]; + const auto& tensorsInBlock = block2tensors[blockIdx]; + int streamIdx = blockIdx % streamList_.size(); + uint8_t* blockAddr = (uint8_t*)block.devPtr; + + for (const auto& tensor : tensorsInBlock) { + uint8_t* tensorStart = (uint8_t*)tensor->devPtr; + uint8_t* tensorEnd = tensorStart + tensor->size; + + // copy the gap between the last tensor(or the start of the block) and the current tensor + size_t lastSegmentSize = tensorStart - blockAddr; + if (lastSegmentSize > 0) { + cudaError_t memcpy_exit_code = cudaMemcpyAsync(serverState_.hostPtr + bufferOffset, (void*)blockAddr, lastSegmentSize, cudaMemcpyDeviceToHost, streamList_[streamIdx]); + if (memcpy_exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync failed for the memory segment in block[%zu] (%p : %p) with %zu bytes: %s\n", blockIdx, blockAddr, tensorStart, lastSegmentSize, cudaGetErrorString(memcpy_exit_code)); + exit(EXIT_FAILURE); + } + else { + bufferOffset += lastSegmentSize; + tool::Logging(LOG_DEBUG, myName, "backup the memory segment in block[%zu] (%p : %p) with %zu bytes\n", blockIdx, blockAddr, tensorStart, lastSegmentSize); + } + } + // move the blockAddr to the end of the current tensor + blockAddr = tensorEnd; + } + + // copy the remaining data in the block + size_t remainingSize = (uint8_t*)block.devPtr+block.size - blockAddr; + if (remainingSize > 0) { + cudaError_t memcpy_exit_code = cudaMemcpyAsync(serverState_.hostPtr + bufferOffset, (void*)blockAddr, remainingSize, cudaMemcpyDeviceToHost, streamList_[streamIdx]); + if (memcpy_exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync failed for the remaining memory segment in block[%zu] (%p) with %zu bytes: %s\n", blockIdx, blockAddr, remainingSize, cudaGetErrorString(memcpy_exit_code)); + tool::Logging(LOG_ERROR, myName, "bufferOffset=%zu, essentialBlockNum=%zu\n", bufferOffset, essentialBlockNum); + exit(EXIT_FAILURE); + } + else { + tool::Logging(LOG_DEBUG, myName, "backup the remaining memory segment in block[%zu] (%p) with %zu bytes\n", blockIdx, blockAddr, remainingSize); + bufferOffset += remainingSize; + essentialBlockNum++; + } + } + } + for (int i = 0; i < streamList_.size(); i++) { + cudaError_t sync_code = cudaStreamSynchronize(streamList_[i]); + if (sync_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaStreamSynchronize failed: %s\n", cudaGetErrorString(sync_code)); + exit(EXIT_FAILURE); + } + } + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration = end - start; + tool::Logging(LOG_INFO, myName, "backup device memory success with %zu bytes (%zu essential blocks) in %f seconds\n", bufferOffset, essentialBlockNum, duration.count()); + + const char* backupcsv_path = std::getenv("FLEXGV_BACKUP_CSV_PATH"); + if (backupcsv_path != NULL) { + std::ofstream outFile; + outFile.open(backupcsv_path, std::ios::app); + if (outFile.tellp() == 0) { + outFile << "Bytes,Time" << std::endl; + } + outFile << bufferOffset << "," << duration.count() << std::endl; + outFile.close(); + } + + // /* Signal the sub thread that the buffer is ready to be written */ + // { + // boost::unique_lock lock(backupSync_.mutex); + // bufferReady_ = true; + // backupSync_.cv.notify_one(); + // } + + stateBackup_.Notify(); + + // debug: + // std::vector& tmpBlocks = _cuInfoMap.blockManager->GetBlocks(); + // for (size_t blockIdx = 0; blockIdx < tmpBlocks.size(); blockIdx++) { + // if (tmpBlocks[blockIdx].valid == false) { + // continue; + // } + // tool::Logging(LOG_DEBUG, myName, "blockInfoList[%zu]: devPtr=%p, size=%zu, valid=%d\n", blockIdx, (void*)tmpBlocks[blockIdx].devPtr, tmpBlocks[blockIdx].size, tmpBlocks[blockIdx].valid); + // cudaFree((void*)tmpBlocks[blockIdx].devPtr); + // tmpBlocks[blockIdx].valid = false; + // tmpBlocks[blockIdx].devPtr = (uint64_t)NULL; + // tmpBlocks[blockIdx].size = 0; + // } + // std::vector& tmpHandles = _cuInfoMap.handleManager->GetHandleInfoList(); + // for (size_t i = 1; i < tmpHandles.size(); i++) { + // if (tmpHandles[i].valid == false) { + // continue; + // } + // tool::Logging(LOG_INFO, myName, "handleInfoList[%zu]: handlePtr=%p, type=%d, valid=%d\n", i, (void*)tmpHandles[i].handlePtr, tmpHandles[i].type, tmpHandles[i].valid); + // } +} + +void ServerEndpoint::Persist2File(const char* fileName, const uint8_t* data, size_t size) { + try { + if (bufferResized_) { + bufferResized_ = false; + /* resize or create the file */ + std::ofstream ofs(fileName, std::ios::binary | std::ios::out | std::ios::trunc); + if (!ofs) { + throw std::runtime_error("Opening file failed: " + std::string(fileName)); + } + ofs.seekp(size - 1); + ofs.write("", 1); + ofs.close(); + tool::Logging(LOG_DEBUG, "Persist2File", "File(%s) is resized to %zu bytes\n", fileName, size); + + /* open the file and map to memory */ + boost::interprocess::file_mapping fileMapping(fileName, boost::interprocess::read_write); + fileMappingRegion_ = std::make_unique(fileMapping, boost::interprocess::read_write, 0, size); + } + + // copy the data to the memory-mapped file + auto start = std::chrono::high_resolution_clock::now(); + std::memcpy(fileMappingRegion_->get_address(), data, size); + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration = end - start; + tool::Logging(LOG_INFO, "Persist2File", "Time taken to copy data to file(%s): %f seconds\n", fileName, duration.count()); + } + catch (const boost::interprocess::interprocess_exception& ex) { + tool::Logging(LOG_ERROR, "Persist2File", "Writing to file failed: %s\n", ex.what()); + } + catch (const std::exception& ex) { + tool::Logging(LOG_ERROR, "Persist2File", "Writing to file failed: general error(%s)\n", ex.what()); + } +} + +void ServerEndpoint::Backup2Storage() { + while (true) { + // /* Wait for main thread to signal that the buffer is ready or the thread should exit */ + // { + // boost::unique_lock lock(backupSync_.mutex); + // backupSync_.cv.wait(lock, [this] { return bufferReady_ || bufferFinished_; }); + // if (bufferFinished_ && !bufferReady_) { + // break; + // } + // } + if (!stateBackup_.Check()) { + break; + } + + /* Write the buffer to storage */ + Persist2File(backupFilePath_.c_str(), serverState_.hostPtr, serverState_.size); + + // /* Signal the main thread that the buffer has been written */ + // { + // boost::unique_lock lock(backupSync_.mutex); + // bufferReady_ = false; + // backupSync_.cv.notify_one(); + // } + + stateBackup_.Notify(); + } +} + +void ServerEndpoint::LoadFromStorage() { + const char* myName = "Load"; + const char* fileName = backupFilePath_.c_str(); + tool::Logging(LOG_DEBUG, myName, "ready to load the memory blocks and the handle mappings.\n"); + if (streamList_.size() == 0) { + streamList_.resize(BACKUP_STREAM_NUM, NULL); + for (int i = 0; i < streamList_.size(); i++) { + cudaStreamCreateWithFlags(&streamList_[i], cudaStreamNonBlocking); + } + } + try { +/* + std::string tmpFileName = backupFilePath_ + ".tmp"; + std::ifstream tmpofs(tmpFileName, std::ios::binary | std::ios::ate); + if (!tmpofs.is_open()) { + throw std::runtime_error(std::string("Unable to open file: ") + tmpFileName); + } + std::streamsize tmpFileSize = tmpofs.tellg(); + if (tmpFileSize < 0) { + throw std::runtime_error("Unable to get file size"); + } + tmpofs.close(); + size_t trainDataSize = static_cast(tmpFileSize); + backupMemcpyBuffer_ = malloc(trainDataSize); + tool::Logging(LOG_DEBUG, myName, "backupMemcpyBuffer_ is allocated with %zu bytes\n", trainDataSize); + boost::interprocess::file_mapping tmpFile(tmpFileName.c_str(), boost::interprocess::read_only); + boost::interprocess::mapped_region tmpRegion(tmpFile, boost::interprocess::read_only, 0, trainDataSize); + std::memcpy(backupMemcpyBuffer_, tmpRegion.get_address(), trainDataSize); + tool::Logging(LOG_INFO, myName, "backupMemcpyBuffer_ is loaded with %zu bytes\n", trainDataSize); +*/ + + /* Get the file size */ + std::ifstream ofs(fileName, std::ios::binary | std::ios::ate); + if (!ofs.is_open()) { + throw std::runtime_error(std::string("Unable to open file: ") + fileName); + } + std::streamsize fileSize = ofs.tellg(); + if (fileSize < 0) { + throw std::runtime_error("Unable to get file size"); + } + ofs.close(); + size_t dataSize = static_cast(fileSize); + + boost::interprocess::file_mapping file(fileName, boost::interprocess::read_only); + boost::interprocess::mapped_region region(file, boost::interprocess::read_only, 0, dataSize); + + size_t bufferOffset = 0; + size_t handleNum = 0, blockNum = 0, memcpyCnt = 0; + memcpy(&lastCopyLen_, (uint8_t*)region.get_address() + bufferOffset, sizeof(lastCopyLen_)); + bufferOffset += sizeof(lastCopyLen_); + memcpy(&lastCopyType_, (uint8_t*)region.get_address() + bufferOffset, sizeof(lastCopyType_)); + bufferOffset += sizeof(lastCopyType_); + memcpy(&handleNum, (uint8_t*)region.get_address() + bufferOffset, sizeof(size_t)); + bufferOffset += sizeof(size_t); + memcpy(&blockNum, (uint8_t*)region.get_address() + bufferOffset, sizeof(size_t)); + bufferOffset += sizeof(size_t); + memcpy(&memcpyCnt, (uint8_t*)region.get_address() + bufferOffset, sizeof(size_t)); + bufferOffset += sizeof(size_t); + tool::Logging(LOG_INFO, myName, "ready to load %zu (valid) handles and %zu blocks\n", handleNum, blockNum); + + // std::vector& tmpHandleList = _cuInfoMap.handleManager->GetHandleInfoList(); + std::vector> tmpHandleList; + tmpHandleList.resize(handleNum); + // std::memcpy(tmpHandleList.data(), (uint8_t*)region.get_address() + bufferOffset, sizeof(Handle_t) * handleNum); + std::memcpy(tmpHandleList.data(), (uint8_t*)region.get_address() + bufferOffset, sizeof(tmpHandleList[0]) * handleNum); + // bufferOffset += sizeof(Handle_t) * handleNum; + bufferOffset += sizeof(tmpHandleList[0]) * handleNum; + _cuInfoMap.handleManager->Reset(tmpHandleList); + tool::Logging(LOG_INFO, myName, "handleInfoList is successfully loaded with %zu handles\n", handleNum); + // _cuInfoMap.handleManager->Indexing(); + // _cuInfoMap.handleManager->Shrink(); + + std::vector& tmpBlockList = _cuInfoMap.blockManager->GetBlocks(); + tmpBlockList.resize(blockNum); + std::memcpy(tmpBlockList.data(), (uint8_t*)region.get_address() + bufferOffset, sizeof(Block_t) * blockNum); + bufferOffset += sizeof(Block_t) * blockNum; + tool::Logging(LOG_INFO, myName, "blockInfoList is successfully loaded with %zu blocks\n", blockNum); + + trainTensors_.resize(memcpyCnt); + std::memcpy(trainTensors_.data(), (uint8_t*)region.get_address() + bufferOffset, sizeof(TensorInfo_t) * memcpyCnt); + bufferOffset += sizeof(TensorInfo_t) * memcpyCnt; + + std::vector sortedTensors = trainTensors_; + std::sort(sortedTensors.begin(), sortedTensors.end(), + [](const TensorInfo_t& a, const TensorInfo_t& b) { + return a.devPtr < b.devPtr; + }); + std::vector> block2tensors(tmpBlockList.size()); + for (size_t i = 0; i < sortedTensors.size(); i++) { + uint64_t tmpAddr = 0; + int blockIdx = _cuInfoMap.blockManager->FindByVirAddr((uint64_t)sortedTensors[i].devPtr, tmpAddr); + if (blockIdx != -1) { + const Block_t& tmpBlock = tmpBlockList[blockIdx]; + if (tmpBlock.essential && (uint8_t*)sortedTensors[i].devPtr + sortedTensors[i].size <= (uint8_t*)tmpBlock.start + tmpBlock.size) { + block2tensors[blockIdx].push_back(&sortedTensors[i]); + } //!: the devPtr of the sortedTensors is virtual address because the real address is not available + } + } + + auto start = std::chrono::high_resolution_clock::now(); + size_t essentialBlockNum = 0; + for (size_t blockIdx = 0; blockIdx < tmpBlockList.size(); blockIdx++) { + if (tmpBlockList[blockIdx].valid == false) { + continue; + } + Block_t* block = &tmpBlockList[blockIdx]; + void* devPtr = NULL; + cudaError_t exit_code = cudaMalloc(&devPtr, block->size); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMalloc failed: %s\n", cudaGetErrorString(exit_code)); + exit(EXIT_FAILURE); + } + else { + block->devPtr = (uint64_t)devPtr; + tool::Logging(LOG_DEBUG, myName, "blockInfoList[%zu]: devPtr=%p (virAddr=%p:%p), size=%zu, valid=%d, essential=%d\n", blockIdx, (void*)block->devPtr, (void*)block->start, (uint8_t*)block->start+block->size, block->size, block->valid, block->essential); + } + + if (tmpBlockList[blockIdx].essential == false) { + continue; + } + essentialBlockNum++; + + const auto& tensorsInBlock = block2tensors[blockIdx]; + int streamIdx = blockIdx % streamList_.size(); + uint8_t* blockAddr = (uint8_t*)block->devPtr; + + for (const auto& tensor: tensorsInBlock) { + uint8_t* tensorStart = (uint8_t*)devPtr + ((uint64_t)tensor->devPtr - block->start); // the real address of the tensor + uint8_t* tensorEnd = tensorStart + tensor->size; + + // copy the gap between the last tensor(or the start of the block) and the current tensor + size_t lastSegmentSize = tensorStart - blockAddr; + if (lastSegmentSize > 0) { + cudaError_t memcpy_exit_code = cudaMemcpyAsync((void*)blockAddr, (uint8_t*)region.get_address() + bufferOffset, lastSegmentSize, cudaMemcpyHostToDevice, streamList_[streamIdx]); + if (memcpy_exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync failed for the memory segment in block[%zu] (%p : %p) with %zu bytes: %s\n", blockIdx, blockAddr, tensorStart, lastSegmentSize, cudaGetErrorString(memcpy_exit_code)); + exit(EXIT_FAILURE); + } + else { + bufferOffset += lastSegmentSize; + tool::Logging(LOG_DEBUG, myName, "load the memory segment in block[%zu] (%p : %p) with %zu bytes\n", blockIdx, blockAddr, tensorStart, lastSegmentSize); + } + } + // move the blockAddr to the end of the current tensor + blockAddr = tensorEnd; + } + // copy the remaining data in the block + size_t remainingSize = (uint8_t*)block->devPtr + block->size - blockAddr; + if (remainingSize > 0) { + cudaError_t memcpy_exit_code = cudaMemcpyAsync((void*)blockAddr, (uint8_t*)region.get_address() + bufferOffset, remainingSize, cudaMemcpyHostToDevice, streamList_[streamIdx]); + if (memcpy_exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync failed for the remaining memory segment in block[%zu] (%p) with %zu bytes: %s\n", blockIdx, blockAddr, remainingSize, cudaGetErrorString(memcpy_exit_code)); + exit(EXIT_FAILURE); + } + else { + tool::Logging(LOG_DEBUG, myName, "load the remaining memory segment in block[%zu] (%p) with %zu bytes\n", blockIdx, blockAddr, remainingSize); + bufferOffset += remainingSize; + } + } + } + + for (int i = 0; i < streamList_.size(); i++) { + cudaError_t sync_code = cudaStreamSynchronize(streamList_[i]); + if (sync_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaStreamSynchronize failed: %s\n", cudaGetErrorString(sync_code)); + exit(EXIT_FAILURE); + } + } + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration = end - start; + tool::Logging(LOG_INFO, myName, "load device memory success with %zu bytes (%zu essential blocks) in %f seconds\n", bufferOffset, essentialBlockNum, duration.count()); + + // debug: + std::vector& tmpBlocks = _cuInfoMap.blockManager->GetBlocks(); + // for (size_t blockIdx = 0; blockIdx < tmpBlocks.size(); blockIdx++) { + // if (tmpBlocks[blockIdx].valid == false) { + // continue; + // } + // tool::Logging(LOG_INFO, myName, "blockInfoList[%zu]: devPtr=%p, size=%zu, valid=%d\n", blockIdx, (void*)tmpBlocks[blockIdx].devPtr, tmpBlocks[blockIdx].size, tmpBlocks[blockIdx].valid); + // } + // std::vector& tmpHandles = _cuInfoMap.handleManager->GetHandleInfoList(); + // for (size_t i = 1; i < tmpHandles.size(); i++) { + // if (tmpHandles[i].valid == false) { + // continue; + // } + // tool::Logging(LOG_INFO, myName, "handleInfoList[%zu]: handlePtr=%p, type=%d, valid=%d\n", i, (void*)tmpHandles[i].handlePtr, tmpHandles[i].type, tmpHandles[i].valid); + // } + } + catch (const boost::interprocess::interprocess_exception& ex) { + tool::Logging(LOG_ERROR, myName, "Reading from file failed: %s\n", ex.what()); + } + catch (const std::exception& ex) { + tool::Logging(LOG_ERROR, myName, "Reading from file failed: general error(%s)\n", ex.what()); + } +} + +void ServerEndpoint::BackupTrainTensors2Storage() { + const char* myName = "BackupTraintensors"; + size_t miniBatchSize = 0; + for (const auto& tensor : trainTensors_) { + miniBatchSize += tensor.size; + } + if (trainTensorsMappingRegion_ == nullptr) { + std::string fileName = backupFilePath_ + ".tmp"; + const char* fileNamecStr = fileName.c_str(); + std::ofstream ofs(fileName, std::ios::binary | std::ios::out | std::ios::trunc); + if (!ofs) { + throw std::runtime_error("Opening file failed: " + fileName); + } + ofs.seekp(miniBatchSize * BACKUP_PERIOD - 1); + ofs.write("", 1); + ofs.close(); + tool::Logging(LOG_DEBUG, myName, "File(%s) is resized to %zu bytes\n", fileNamecStr, miniBatchSize * BACKUP_PERIOD); + + /* open the file and map to memory */ + boost::interprocess::file_mapping fileMapping(fileNamecStr, boost::interprocess::read_write); + trainTensorsMappingRegion_ = std::make_unique(fileMapping, boost::interprocess::read_write, 0, miniBatchSize * BACKUP_PERIOD); + } + + while (true) { + if (!trainTensorBackup_.Check()) { + break; + } + + tool::Logging(LOG_DEBUG, myName, "start to backup the train tensors for iteration#%zu\n", curIter_); + + auto start = std::chrono::high_resolution_clock::now(); + size_t offset = (curIter_ % BACKUP_PERIOD) * miniBatchSize; + for (size_t i = 0; i < trainTensors_.size(); i++) { + int streamIdx = i % streamList_.size(); + // copy the data to the memory-mapped file + cudaError_t exit_code = cudaMemcpyAsync((uint8_t*)trainTensorsMappingRegion_->get_address() + offset, GetDevPtr((uint64_t)trainTensors_[i].devPtr), trainTensors_[i].size, cudaMemcpyDeviceToHost, streamList_[streamIdx]); // backup in the non-blocking stream + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpyAsync failed for the memory segment in tensor[%zu] (%p) with %zu bytes: %s\n", i, trainTensors_[i].devPtr, trainTensors_[i].size, cudaGetErrorString(exit_code)); + exit(EXIT_FAILURE); + } + else { + tool::Logging(LOG_DEBUG, myName, "backup the memory segment in tensor[%zu] (%p) with %zu bytes\n", i, trainTensors_[i].devPtr, trainTensors_[i].size); + + cudaError_t sync_code = cudaStreamSynchronize(streamList_[streamIdx]); + if (sync_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaStreamSynchronize failed: %s\n", cudaGetErrorString(sync_code)); + exit(EXIT_FAILURE); + } + } + offset += trainTensors_[i].size; + } + auto end = std::chrono::high_resolution_clock::now(); + tool::Logging(LOG_DEBUG, myName, "backup the train tensors for iteration#%zu in %f seconds\n", curIter_, std::chrono::duration(end - start).count()); + + trainTensorBackup_.Notify(); + } +} + +void ServerEndpoint::LoadFromStorage(size_t iter) { + const char* myName = "LoadTrainTensors"; + size_t miniBatchSize = 0; + for (const auto& tensor : trainTensors_) { + miniBatchSize += tensor.size; + } + if (trainTensorsMappingRegion_ == nullptr) { + std::string fileName = backupFilePath_ + ".tmp"; + std::ifstream ofs(fileName, std::ios::binary | std::ios::ate); + if (!ofs.is_open()) { + throw std::runtime_error(std::string("Unable to open file: ") + fileName); + } + std::streamsize fileSize = ofs.tellg(); + if (fileSize < 0) { + throw std::runtime_error("Unable to get file size"); + } + ofs.close(); + size_t dataSize = static_cast(fileSize); + + boost::interprocess::file_mapping fileMapping(fileName.c_str(), boost::interprocess::read_write); + trainTensorsMappingRegion_ = std::make_unique(fileMapping, boost::interprocess::read_write, 0, dataSize); + } + size_t offset = (iter % BACKUP_PERIOD) * miniBatchSize; + for (const auto& tensor : trainTensors_) { + cudaError_t exit_code = cudaMemcpyAsync(GetDevPtr((uint64_t)tensor.devPtr), (uint8_t*)trainTensorsMappingRegion_->get_address() + offset, tensor.size, cudaMemcpyHostToDevice, defaultStream_); // loading in the default stream because it is in the critical path + offset += tensor.size; + + cudaError_t sync_code = cudaStreamSynchronize(defaultStream_); + if (sync_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaStreamSynchronize failed: %s\n", cudaGetErrorString(sync_code)); + exit(EXIT_FAILURE); + } + } + tool::Logging(LOG_INFO, myName, "load the train tensors for iteration#%zu\n", iter); +} + +void ServerEndpoint::StopBackup() { + // // notify the sub thread to exit + // { + // boost::unique_lock lock(backupSync_.mutex); + // bufferReady_ = false; + // bufferFinished_ = true; + // backupSync_.cv.notify_one(); + // } + // backupStorageThread_->join(); + + cudaFreeHost(serverState_.hostPtr); + cudaFreeHost(backupMemcpyBuffer_); + for (int i = 0; i < streamList_.size(); i++) { + cudaStreamDestroy(streamList_[i]); + } +} + +void ServerEndpoint::CommEventMonitor() { + while (true) { + std::list tmpList; + + boost::unique_lock lock(eventWatchedSync_.mutex); + if (eventWatchedSync_.cv.wait_for(lock, boost::chrono::seconds(COMM_EVENT_TIMEOUT), + [this] { return bufferFinished_;})) { + tool::Logging(LOG_INFO, "CommEventMonitor", "stop watching events\n"); + break; + } + + tmpList = watchedEventsList_; + if (!notCompleteEventsList_.empty()) { + tool::Logging(LOG_INFO, "CommEventMonitor", "watching %zu events (not complete in last round)\n", notCompleteEventsList_.size()); + for (const auto& elem : notCompleteEventsList_) { + Handle_t* eventHandle = _cuInfoMap.handleManager->GetHandleInfoByRealAddr(elem); + if (eventHandle == NULL || eventHandle->valid == false) { + continue; + } + + cudaError_t exit_code = cudaEventQuery(elem); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, "CommEventMonitor", "cudaEventQuery failed for event(%p): %s\n", (void*)elem, cudaGetErrorString(exit_code)); + // ncclCommAbort(curComm); // trigger segmentation fault + std::terminate(); // as stucking in the callback function, so need to use this to exit + + break; + } + } + // Normally, notCompleteEventsList_ should be empty after the above loop + notCompleteEventsList_.clear(); + } + + if (tmpList.empty()) { + continue; + } + tool::Logging(LOG_INFO, "CommEventMonitor", "watching %zu events\n", tmpList.size()); + for (const auto& elem : tmpList) { + watchedEventsList_.remove(elem); + Handle_t* eventHandle = _cuInfoMap.handleManager->GetHandleInfoByRealAddr(elem); + tool::Logging(LOG_DEBUG, "CommEventMonitor", "watching event(%p from %p)\n", (void*)elem, (void*)eventHandle); + + if (eventHandle == NULL || eventHandle->valid == false) { + // cudaEventDestroy(elem); // destroyed by cudaEventDestroyHandle + continue; + } + + cudaError_t exit_code = cudaEventQuery(elem); + if (exit_code == cudaSuccess) { + // cudaEventDestroy(elem); // destroyed by cudaEventDestroyHandle + } + else { + notCompleteEventsList_.push_back(elem); + } + } + } +} + +void ServerEndpoint::StopCommEventMonitor() { + { + boost::unique_lock lock(eventWatchedSync_.mutex); + bufferFinished_ = true; + eventWatchedSync_.cv.notify_one(); + } + eventWatchedThread_->join(); +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/server/serverEndpoint.cc b/GPU-Virtual-Service/gpu-remoting/src/server/serverEndpoint.cc new file mode 100644 index 0000000..4532855 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/server/serverEndpoint.cc @@ -0,0 +1,513 @@ +#include "../../include/serverEndpoint.h" + +ServerEndpoint::ServerEndpoint(ucp_worker_h dataWorker, UCPConnection_t conn){ + _dataWorker = dataWorker; + _connectReq = conn.conn_request; + clientID_ = conn.client_id; + clientIP_ = conn.client_ip; + clientPort_ = conn.client_port; + string wholeName = myShortName_ + std::string(" ") + clientIP_ + std::string(":") + clientPort_; + myName_ = strdup(wholeName.c_str()); + + _cuInfoMap.isFirstIter = true; + _cuInfoMap.mapFatBinHandle2CuModule = new robin_hood::unordered_flat_map(); + _cuInfoMap.mapHost2CuFunc = new robin_hood::unordered_flat_map(); + _cuInfoMap.mapDevName2DevPtr = new robin_hood::unordered_flat_map(); + _cuInfoMap.mapHostVar2CuDevPtr = new robin_hood::unordered_flat_map(); + +#ifdef GV_HANDLE + _cuInfoMap.handleManager = new HandleMapper(); +#endif // GV_HANDLE + +#ifdef GV_MEMORY + _cuInfoMap.blockManager = new DeviceBlockMapper(); +#else + _cuInfoMap.blockInfoList.reserve(1000); + _cuInfoMap.blockInfoList.emplace_back(Block_t{.devPtr = (uint64_t)NULL, .size = 0, .valid = true}); // let the blockNum start from 1 +#endif // GV_MEMORY + + // _cuInfoMap.ncclUidList.reserve(5); + // _cuInfoMap.ncclUidList.emplace_back(ncclUniqueId{}); // let the ncclUidNum start from 1 + _cuInfoMap.ncclRedOpList.resize(ncclNumOps + 1, static_cast(0)); + for (int i = 0; i < ncclNumOps; i++) { + _cuInfoMap.ncclRedOpList[i] = static_cast(i); + } + + + /* __cudaRegister will initialize shmQueSizes_, imageQueue and ptxExtractor */ + _cuInfoMap.imageQueue = nullptr; + _cuInfoMap.ptxExtractor = nullptr; + _fatbinList.reserve(240); + + trainTensors_.reserve(5); + boost::thread_attributes attrs; + attrs.set_stack_size(THREAD_STACK_SIZE); +#ifdef GV_BACKUP + stateBackup_.Start(boost::bind(&ServerEndpoint::Backup2Storage, this)); + // backupStorageThread_ = new boost::thread(attrs, boost::bind(&ServerEndpoint::Backup2Storage, this)); + eventWatchedThread_ = new boost::thread(attrs, boost::bind(&ServerEndpoint::CommEventMonitor, this)); +#endif // GV_BACKUP + +#ifdef GV_Scheduler + sche = new Scheduler(clientID_); +#endif + +#ifdef GV_eScheduler + esche = new eScheduler(clientID_); +#endif + +#ifdef GV_MSGHANDLER + msghandler = new MsgHandler(clientID_); +#endif + + /* The client side should have initiated the connection, leading to this ep's creation */ + ucp_ep_params_t ep_params; + ucs_status_t status; + ep_params.field_mask = UCP_EP_PARAM_FIELD_ERR_HANDLER | + UCP_EP_PARAM_FIELD_CONN_REQUEST; + ep_params.conn_request = _connectReq; + ep_params.err_handler.cb = ErrorCallback; + ep_params.err_handler.arg = &connStatus_; + if ((status = ucp_ep_create(_dataWorker, &ep_params, &_clientEp)) != UCS_OK) { + tool::Logging(LOG_ERROR, myName_, "failed to create an endpoint (%s)\n", ucs_status_string(status)); + ucp_worker_destroy(_dataWorker); + free(_connectReq); // todo + } + + tool::Logging(LOG_COMM, myName_, "create the Server endpoint object (connected to client#%llu %s:%s).\n", clientID_, clientIP_, clientPort_); +} + +void ServerEndpoint::NewRun() { + tool::Logging(LOG_INFO, myName_, "start the server endpoint to serve the client#%llu.\n", clientID_); + connStatus_.isClosed = false; + + RegisterHandler(__CUDA_REGISTER, __cudaRegisterHandle, _dataWorker, this); + RegisterHandler(__CUDA_REGISTER_FAT_BINARY, __cudaRegisterFatBinaryHandle, _dataWorker, this); + RegisterHandler(__CUDA_REGISTER_FUNCTION, __cudaRegisterFunctionHandle, _dataWorker, this); + RegisterHandler(__CUDA_REGISTER_VAR, __cudaRegisterVarHandle, _dataWorker, this); + RegisterHandler(CUDA_LAUNCH_KERNEL, cudaLaunchKernelHandle, _dataWorker, this); + RegisterHandler(CUDA_FUNC_GET_ATTRIBUTES, cudaFuncGetAttributesHandle, _dataWorker, this); + RegisterHandler(CUDA_MALLOC, cudaMallocHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMSET, cudaMemsetHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMSET_ASYNC, cudaMemsetAsyncHandle, _dataWorker, this); + RegisterHandler(CUDA_MEM_GET_INFO, cudaMemGetInfoHandle, _dataWorker, this); + RegisterHandler(CUDA_FREE, cudaFreeHandle, _dataWorker, this); + // RegisterHandler(CUDA_MEMCPY, cudaMemcpyHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMCPY_H2D, cudaMemcpyH2DHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMCPY_D2H, cudaMemcpyD2HHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMCPY_D2D, cudaMemcpyD2DHandle, _dataWorker, this); + RegisterHandler(NEW_ITERATION_REQ, NewIterHandle, _dataWorker, this); + // RegisterHandler(CUDA_MEMCPY_ASYNC, cudaMemcpyAsyncHandle, _dataWorker, this); //todo + RegisterHandler(CUDA_MEMCPY_ASYNC_H2D, cudaMemcpyAsyncH2DHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMCPY_ASYNC_D2H, cudaMemcpyAsyncD2HHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMCPY_ASYNC_D2D, cudaMemcpyAsyncD2DHandle, _dataWorker, this); + RegisterHandler(CUDA_MEMCPY_TO_SYMBOL, cudaMemcpyToSymbolHandle, _dataWorker, this); + RegisterHandler(CUDA_DEVICE_SYNCHRONIZE, cudaDeviceSynchronizeHandle, _dataWorker, this); + RegisterHandler(CUDA_GET_DEVICE, cudaGetDeviceHandle, _dataWorker, this); + RegisterHandler(CUDA_GET_DEVICE_COUNT, cudaGetDeviceCountHandle, _dataWorker, this); + RegisterHandler(CUDA_SET_MAIN_DEVICE, cudaSetMainDeviceHandle, _dataWorker, this); + RegisterHandler(CUDA_SET_DEVICE, cudaSetDeviceHandle, _dataWorker, this); + RegisterHandler(CUDA_GET_DEVICE_PROPERTIES, cudaGetDevicePropertiesHandle, _dataWorker, this); + RegisterHandler(CUDA_DEVICE_GET_ATTRIBUTE, cudaDeviceGetAttributeHandle, _dataWorker, this); + RegisterHandler(CUDA_EVENT_CREATE, cudaEventCreateHandle, _dataWorker, this); + RegisterHandler(CUDA_EVENT_CREATE_WITH_FLAGS, cudaEventCreateWithFlagsHandle, _dataWorker, this); + RegisterHandler(CUDA_EVENT_RECORD, cudaEventRecordHandle, _dataWorker, this); + RegisterHandler(CUDA_EVENT_QUERY, cudaEventQueryHandle, _dataWorker, this); + RegisterHandler(CUDA_EVENT_DESTROY, cudaEventDestroyHandle, _dataWorker, this); + RegisterHandler(CUDA_EVENT_ELAPSED_TIME, cudaEventElapsedTimeHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_DESTROY, cudaStreamDestroyHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_CREATE, cudaStreamCreateHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_CREATE_WITH_FLAGS, cudaStreamCreateWithFlagsHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_CREATE_WITH_PRIORITY, cudaStreamCreateWithPriorityHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_WAIT_EVENT, cudaStreamWaitEventHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_SYNCHRONIZE, cudaStreamSynchronizeHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_IS_CAPTURING, cudaStreamIsCapturingHandle, _dataWorker, this); + RegisterHandler(CUDA_STREAM_GET_CAPTURE_INFO, cudaStreamGetCaptureInfoHandle, _dataWorker, this); + RegisterHandler(CUDA_OCCUPANCY_MAX_ACTIVE_BLOCKS_PER_MULTIPROCESSOR, cudaOccupancyMaxActiveBlocksPerMultiprocessorHandle, _dataWorker, this); + RegisterHandler(CUDA_OCCUPANCY_MAX_ACTIVE_BLOCKS_PER_MULTIPROCESSOR_WITH_FLAGS, cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlagsHandle, _dataWorker, this); + + + RegisterHandler(CUBLAS_CREATE_V2, cublasCreate_v2Handle, _dataWorker, this); + RegisterHandler(CUBLAS_SGEMM_V2, cublasSgemm_v2Handle, _dataWorker, this); + RegisterHandler(CUBLAS_SGEMM_STRIDED_BATCHED, cublasSgemmStridedBatchedHandle, _dataWorker, this); + RegisterHandler(CUBLAS_DESTROY_V2, cublasDestroy_v2Handle, _dataWorker, this); + RegisterHandler(CUBLAS_SET_STREAM_V2, cublasSetStream_v2Handle, _dataWorker, this); + RegisterHandler(CUBLAS_SET_WORKSPACE_V2, cublasSetWorkspace_v2Handle, _dataWorker, this); + RegisterHandler(CUBLAS_SET_MATH_MODE, cublasSetMathModeHandle, _dataWorker, this); + RegisterHandler(CUBLAS_GET_MATH_MODE, cublasGetMathModeHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_CREATE, cublasLtCreateHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_DESTROY, cublasLtDestroyHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULDESC_CREATE, cublasLtMatmulDescCreateHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULDESC_DESTROY, cublasLtMatmulDescDestroyHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULDESC_SETATTRIBUTE, cublasLtMatmulDescSetAttributeHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATRIX_LAYOUT_CREATE, cublasLtMatrixLayoutCreateHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATRIX_LAYOUT_DESTROY, cublasLtMatrixLayoutDestroyHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATRIX_LAYOUT_SETATTRIBUTE, cublasLtMatrixLayoutSetAttributeHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULPREFERENCE_CREATE, cublasLtMatmulPreferenceCreateHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULPREFERENCE_DESTROY, cublasLtMatmulPreferenceDestroyHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULPREFERENCE_SETATTRIBUTE, cublasLtMatmulPreferenceSetAttributeHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMULALGO_GETHEURISTIC, cublasLtMatmulAlgoGetHeuristicHandle, _dataWorker, this); + RegisterHandler(CUBLASLT_MATMUL, cublasLtMatmulHandle, _dataWorker, this); + + + RegisterHandler(CUDNN_CREATE, cudnnCreateHandle, _dataWorker, this); + RegisterHandler(CUDNN_DESTROY, cudnnDestroyHandle, _dataWorker, this); + RegisterHandler(CUDNN_CREATE_TENSOR_DESCRIPTOR, cudnnCreateTensorDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_DESTROY_TENSOR_DESCRIPTOR, cudnnDestroyTensorDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_TENSOR_SIZE_IN_BYTES, cudnnGetTensorSizeInBytesHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_TENSOR_4D_DESCRIPTOR, cudnnSetTensor4dDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_TENSOR_ND_DESCRIPTOR, cudnnSetTensorNdDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_TENSOR_ND_DESCRIPTOR_EX, cudnnSetTensorNdDescriptorExHandle, _dataWorker, this); + RegisterHandler(CUDNN_CREATE_TENSOR_TRANSFORM_DESCRIPTOR, cudnnCreateTensorTransformDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_TENSOR_TRANSFORM_DESCRIPTOR, cudnnSetTensorTransformDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_DESTROY_TENSOR_TRANSFORM_DESCRIPTOR, cudnnDestroyTensorTransformDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_INIT_TRANSFORM_DEST, cudnnInitTransformDestHandle, _dataWorker, this); + RegisterHandler(CUDNN_TRANSFORM_TENSOR_EX, cudnnTransformTensorExHandle, _dataWorker, this); + RegisterHandler(CUDNN_TRANSFORM_FILTER, cudnnTransformFilterHandle, _dataWorker, this); + RegisterHandler(CUDNN_CREATE_FILTER_DESCRIPTOR, cudnnCreateFilterDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_FILTER_ND_DESCRIPTOR, cudnnSetFilterNdDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_DESTROY_FILTER_DESCRIPTOR, cudnnDestroyFilterDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_FILTER_SIZE_IN_BYTES, cudnnGetFilterSizeInBytesHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_FOLDED_CONV_BACKWARD_DATA_DESCRIPTORS, cudnnGetFoldedConvBackwardDataDescriptorsHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_STREAM, cudnnSetStreamHandle, _dataWorker, this); + RegisterHandler(CUDNN_BATCH_NORMALIZATION_BACKWARD_EX, cudnnBatchNormalizationBackwardExHandle, _dataWorker, this); + RegisterHandler(CUDNN_BATCH_NORMALIZATION_FORWARD_TRAINING_EX, cudnnBatchNormalizationForwardTrainingExHandle, _dataWorker, this); + RegisterHandler(CUDNN_BATCH_NORMALIZATION_FORWARD_INFERENCE, cudnnBatchNormalizationForwardInferenceHandle, _dataWorker, this); + RegisterHandler(CUDNN_BACKEND_CREATE_DESCRIPTOR, cudnnBackendCreateDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_BACKEND_DESTROY_DESCRIPTOR, cudnnBackendDestroyDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_BACKEND_SET_ATTRIBUTE, cudnnBackendSetAttributeHandle, _dataWorker, this); + RegisterHandler(CUDNN_BACKEND_GET_ATTRIBUTE, cudnnBackendGetAttributeHandle, _dataWorker, this); + RegisterHandler(CUDNN_BACKEND_EXECUTE, cudnnBackendExecuteHandle, _dataWorker, this); + RegisterHandler(CUDNN_BACKEND_FINALIZE, cudnnBackendFinalizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_BATCH_NORMALIZATION_BACKWARD_EX_WORKSPACE_SIZE, cudnnGetBatchNormalizationBackwardExWorkspaceSizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_BATCH_NORMALIZATION_FORWARD_TRAINING_EX_WORKSPACE_SIZE, cudnnGetBatchNormalizationForwardTrainingExWorkspaceSizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_BATCH_NORMALIZATION_TRAINING_EX_RESERVE_SPACE_SIZE, cudnnGetBatchNormalizationTrainingExReserveSpaceSizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_CREATE_CONVOLUTION_DESCRIPTOR, cudnnCreateConvolutionDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_DESTROY_CONVOLUTION_DESCRIPTOR, cudnnDestroyConvolutionDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_CONVOLUTION_GROUP_COUNT, cudnnSetConvolutionGroupCountHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_CONVOLUTION_MATH_TYPE, cudnnSetConvolutionMathTypeHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_CONVOLUTION_ND_DESCRIPTOR, cudnnSetConvolutionNdDescriptorHandle, _dataWorker, this); + RegisterHandler(CUDNN_SET_CONVOLUTION_REORDER_TYPE, cudnnSetConvolutionReorderTypeHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_CONVOLUTION_FORWARD_ALGORITHM_V7, cudnnGetConvolutionForwardAlgorithm_v7Handle, _dataWorker, this); + RegisterHandler(CUDNN_GET_CONVOLUTION_BACKWARD_FILTER_ALGORITHM_V7, cudnnGetConvolutionBackwardFilterAlgorithm_v7Handle, _dataWorker, this); + RegisterHandler(CUDNN_GET_CONVOLUTION_BACKWARD_DATA_ALGORITHM_V7, cudnnGetConvolutionBackwardDataAlgorithm_v7Handle, _dataWorker, this); + RegisterHandler(CUDNN_GET_CONVOLUTION_FORWARD_WORKSPACE_SIZE, cudnnGetConvolutionForwardWorkspaceSizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_CONVOLUTION_FORWARD, cudnnConvolutionForwardHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_CONVOLUTION_BACKWARD_DATA_WORKSPACE_SIZE, cudnnGetConvolutionBackwardDataWorkspaceSizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_CONVOLUTION_BACKWARD_FILTER, cudnnConvolutionBackwardFilterHandle, _dataWorker, this); + RegisterHandler(CUDNN_GET_CONVOLUTION_BACKWARD_FILTER_WORKSPACE_SIZE, cudnnGetConvolutionBackwardFilterWorkspaceSizeHandle, _dataWorker, this); + RegisterHandler(CUDNN_CONVOLUTION_BACKWARD_DATA, cudnnConvolutionBackwardDataHandle, _dataWorker, this); + RegisterHandler(NCCL_GROUP_START, ncclGroupStartHandle, _dataWorker, this); + RegisterHandler(NCCL_GROUP_END, ncclGroupEndHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_INIT_RANK, ncclCommInitRankHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_DESTROY, ncclCommDestroyHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_GET_ASYNC_ERROR, ncclCommGetAsyncErrorHandle, _dataWorker, this); + RegisterHandler(NCCL_GET_UNIQUE_ID, ncclGetUniqueIdHandle, _dataWorker, this); + RegisterHandler(NCCL_GET_VERSION, ncclGetVersionHandle, _dataWorker, this); + RegisterHandler(NCCL_ALL_REDUCE, ncclAllReduceHandle, _dataWorker, this); + RegisterHandler(NCCL_REDUCE, ncclReduceHandle, _dataWorker, this); + RegisterHandler(NCCL_REDUCE_SCATTER, ncclReduceScatterHandle, _dataWorker, this); + RegisterHandler(NCCL_ALL_GATHER, ncclAllGatherHandle, _dataWorker, this); + RegisterHandler(NCCL_BROADCAST, ncclBroadcastHandle, _dataWorker, this); + RegisterHandler(NCCL_SEND, ncclSendHandle, _dataWorker, this); + RegisterHandler(NCCL_RECV, ncclRecvHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_COUNT, ncclCommCountHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_USER_RANK, ncclCommUserRankHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_CU_DEVICE, ncclCommCuDeviceHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_ABORT, ncclCommAbortHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_INIT_ALL, ncclCommInitAllHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_INIT_RANK_CONFIG, ncclCommInitRankConfigHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_SPLIT, ncclCommSplitHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_FINALIZE, ncclCommFinalizeHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_REGISTER, ncclCommRegisterHandle, _dataWorker, this); + RegisterHandler(NCCL_COMM_DEREGISTER, ncclCommDeregisterHandle, _dataWorker, this); + RegisterHandler(NCCL_MEM_ALLOC, ncclMemAllocHandle, _dataWorker, this); + RegisterHandler(NCCL_MEM_FREE, ncclMemFreeHandle, _dataWorker, this); + RegisterHandler(NCCL_RED_OP_CREATE_PRE_MUL_SUM, ncclRedOpCreatePreMulSumHandle, _dataWorker, this); + RegisterHandler(NCCL_RED_OP_DESTROY, ncclRedOpDestroyHandle, _dataWorker, this); + + while (!connStatus_.isClosed) { + uint res = ucp_worker_progress(_dataWorker); +#ifdef GV_Scheduler + if(res != 0 && CliDev_ != -1){ + sche->enqueue(clientID_, CliDev_); +#ifdef GV_Monitor + if(getDeviceFlag == 0){ + std::string message = "client_runjob:"; + sche->send_message(message,CliDev_); + getDeviceFlag = 1; + } +#endif + } +#endif + +#ifdef GV_MSGHANDLER + if(res != 0 && CliDev_ != -1 && replayFlag_ && !createThreadFlag){ + msghandler->replay_flag = true; + createThreadFlag = true; + boost::thread_attributes attrs; + attrs.set_stack_size(THREAD_STACK_SIZE); + MSGRecvThread_ = new boost::thread(attrs, boost::bind(&MsgHandler::aysnc_receive_message, msghandler)); + MSGRecvThread_->detach(); + } + MSGStopFlag_ = msghandler->stop_flag; + +#endif + +#ifdef GV_eScheduler + if(res != 0 && CliDev_ != -1){ + esche->enqueue(clientID_, CliDev_, priority_); + } +#endif + + } + + // this->~ServerEndpoint(); + delete this; +} + +DEFINE_SERVER_AM_CALLBACK(NewIterHandle) { + const char* myName = "NewIteration"; + ServerEndpoint* serverEp = (ServerEndpoint*) arg; + RequestIOV reqBuf = RequestIOV(header, header_length, data); + size_t tensorNum = reqBuf.Pop(); + TensorInfo_t* tensors = reqBuf.AssignAddrForAll(); + serverEp->trainTensors_.clear(); + size_t totalTensorSize = 0; + for (size_t i = 0; i < tensorNum; i++) { + serverEp->trainTensors_.push_back(tensors[i]); + totalTensorSize += tensors[i].size; + } + + serverEp->curIter_ ++; + tool::Logging(LOG_DEBUG, myName, "client#%llu starts the iteration#%zu.\n", serverEp->clientID_, serverEp->curIter_); + if (serverEp->curIter_ > 1 && serverEp->_cuInfoMap.isFirstIter) { + serverEp->_cuInfoMap.isFirstIter = false; + } + +#ifdef GV_eScheduler + serverEp->esche->cal_add_It(serverEp->clientID_); +#endif + +#ifdef GV_BACKUP + if (serverEp->curIter_ > 0 && serverEp->curIter_ % BACKUP_PERIOD == 0 + && !serverEp->recoveryFlag_) { // avoid backup in replay mode + auto start_back = std::chrono::high_resolution_clock::now(); + serverEp->Backup2Memory(); + auto end_back = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration_back = end_back - start_back; + tool::Logging(myName, "backup total time=%f s\n", duration_back.count()); + serverEp->ckptCnt++; // debug + } + + if (serverEp->curIter_ >= BACKUP_PERIOD && !serverEp->recoveryFlag_) { + // serverEp->Backup2Storage(serverEp->curIter_); + auto start = std::chrono::high_resolution_clock::now(); + if (!serverEp->trainTensorBackup_.CheckStart()) { + serverEp->trainTensorBackup_.Start(boost::bind(&ServerEndpoint::BackupTrainTensors2Storage, serverEp)); + } + serverEp->trainTensorBackup_.Notify(); + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration = end - start; + tool::Logging(LOG_DEBUG, myName, "submit the backup task for iteration#%zu in %f seconds.\n", serverEp->curIter_, duration.count()); + } + else if (serverEp->recoveryFlag_) { + serverEp->LoadFromStorage(serverEp->curIter_ - 1); // curIter_ starts from 1 while restoring + } + + +/* + if (serverEp->curIter_ == BACKUP_PERIOD && !serverEp->recoveryFlag_) { + cudaError_t exit_code = cudaMallocHost(&serverEp->backupMemcpyBuffer_, totalTensorSize * BACKUP_PERIOD); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMallocHost failed for backup memcpy buffer: %s\n", cudaGetErrorString(exit_code)); + exit(EXIT_FAILURE); + } + } + if (serverEp->backupMemcpyBuffer_ && !serverEp->recoveryFlag_) { + auto start = std::chrono::high_resolution_clock::now(); + size_t offset = (serverEp->curIter_ % BACKUP_PERIOD) * totalTensorSize; // buffer starts from curIter_ = 0 + for (size_t i = 0; i < tensorNum; i++) { + int streamIdx = i % BACKUP_STREAM_NUM; + // cudaError_t exit_code = cudaMemcpy((uint8_t*)serverEp->backupMemcpyBuffer_ + offset, serverEp->GetDevPtr((uint64_t)tensors[i].devPtr), tensors[i].size, cudaMemcpyDeviceToHost); + cudaError_t exit_code = cudaMemcpyAsync((uint8_t*)serverEp->backupMemcpyBuffer_ + offset, serverEp->GetDevPtr((uint64_t)tensors[i].devPtr), tensors[i].size, cudaMemcpyDeviceToHost, serverEp->streamList_[streamIdx]); + if (exit_code != cudaSuccess) { + tool::Logging(LOG_ERROR, myName, "cudaMemcpy failed for tensor[%zu](%p) with size %zu: %s\n", i, tensors[i].devPtr, tensors[i].size, cudaGetErrorString(exit_code)); + exit(EXIT_FAILURE); + } + else { + tool::Logging(LOG_DEBUG, myName, "cudaMemcpy for tensor[%zu](%p) with size %zu in stream %p.\n", i, tensors[i].devPtr, tensors[i].size, serverEp->streamList_[streamIdx]); + } + + offset += tensors[i].size; + } + auto end = std::chrono::high_resolution_clock::now(); + tool::Logging(LOG_DEBUG, myName, "cudaMemcpy for %zu bytes takes %f seconds\n", totalTensorSize, std::chrono::duration(end - start).count()); + } + if (serverEp->recoveryFlag_) { + size_t offset = ((serverEp->curIter_ - 1) % BACKUP_PERIOD) * totalTensorSize; // curIter_ starts from 1 + for (size_t i = 0; i < tensorNum; i++) { + cudaError_t exit_code = cudaMemcpyAsync(serverEp->GetDevPtr((uint64_t)tensors[i].devPtr), (uint8_t*)serverEp->backupMemcpyBuffer_ + offset, tensors[i].size, cudaMemcpyHostToDevice, serverEp->defaultStream_); + // cudaMemcpy(serverEp->GetDevPtr((uint64_t)tensors[i].devPtr), (uint8_t*)serverEp->backupMemcpyBuffer_ + offset, tensors[i].size, cudaMemcpyHostToDevice); + offset += tensors[i].size; + } + // cudaStreamSynchronize(serverEp->defaultStream_); + tool::Logging(LOG_DEBUG, myName, "recovery the tensors for iteration#%zu in stream %p.\n", serverEp->curIter_, serverEp->defaultStream_); + } +*/ + +#endif // GV_BACKUP + + return UCS_OK; +} + +void ServerEndpoint::CreateServerEp(ucp_conn_request_h conn_request, void *arg) { + const char* myName = "CreateServerEp"; + UCPConnection* connectionObj = (UCPConnection*)arg; + UCPConnection_t conn; + + char* ip_str = (char*)malloc(IP_STRING_LEN); + char* port_str = (char*)malloc(PORT_STRING_LEN); + ucp_conn_request_attr_t attr; + attr.field_mask = UCP_CONN_REQUEST_ATTR_FIELD_CLIENT_ADDR | UCP_CONN_REQUEST_ATTR_FIELD_CLIENT_ID; + ucs_status_t status = ucp_conn_request_query(conn_request, &attr); + if (status == UCS_OK) { // get the client address from the connection request + tool::GetIpStrFromSockaddr(&attr.client_address, ip_str, IP_STRING_LEN); + tool::GetPortStrFromSockaddr(&attr.client_address, port_str, PORT_STRING_LEN); + tool::Logging(LOG_INFO, myName, "recv a connection request(%s:%s) from client#%llu.\n", ip_str, port_str, attr.client_id); + conn.client_id = attr.client_id; + conn.client_ip = ip_str; + conn.client_port = port_str; + conn.conn_request = conn_request; + + } else if (status != UCS_ERR_UNSUPPORTED) { + tool::Logging(LOG_ERROR, myName, "failed to query the connection request (%s)\n", ucs_status_string(status)); // todo: UCS_ERR_UNSUPPORTED + exit(EXIT_FAILURE); + } + + ServerEndpoint* serverEndPointObj = new ServerEndpoint(connectionObj->CreateWorker(), conn); + + boost::thread_attributes attrs; + attrs.set_stack_size(THREAD_STACK_SIZE); + boost::thread* thTmp = new boost::thread(attrs, boost::bind(&ServerEndpoint::NewRun, serverEndPointObj)); + thTmp->detach(); + delete thTmp; +} + + +/** + * Close UCP endpoint. + */ +void ServerEndpoint::CloseEp(uint64_t flags) { + ucp_request_param_t param; + ucs_status_t status; + void *close_req; + + param.op_attr_mask = UCP_OP_ATTR_FIELD_FLAGS; + param.flags = flags; + close_req = ucp_ep_close_nbx(_clientEp, ¶m); + if (UCS_PTR_IS_PTR(close_req)) { + do { + ucp_worker_progress(_dataWorker); + status = ucp_request_check_status(close_req); + } while (status == UCS_INPROGRESS); + ucp_request_free(close_req); + } else { + status = UCS_PTR_STATUS(close_req); + } + + if (status != UCS_OK) { + fprintf(stderr, "failed to close ep %p: %s\n", (void*)_clientEp, + ucs_status_string(status)); + } +} + +ServerEndpoint::~ServerEndpoint() { + CloseEp(UCP_EP_CLOSE_MODE_FLUSH); + ucp_worker_destroy(_dataWorker); + //free(_connectReq); + //ucp_listener_reject(_serverConnection.listener, _serverConnection.conn_request); + // if (UCS_PTR_IS_PTR(_connectReq)) { + // ucp_request_free(_connectReq); + // } //! In multi-threaded version, it may cause the error + +#ifdef GV_Scheduler + if(CliDev_ != -1){ + sche->free_jobs(clientID_, CliDev_); +#ifdef GV_Monitor + std::string message = "client_stop:"; + sche->send_message(message,CliDev_); +#endif + } +#endif + +#ifdef GV_eScheduler + if(CliDev_ != -1){ + esche->free_jobs(clientID_, CliDev_); + } + delete esche; +#endif + + + for (int i = 0; i < 3; i++) { + delete shmQueSizes_[i]; + delete shmQueues_[i]; + } + +#ifdef GV_MSGHANDLER + // MSGRecvThread_->join(); + delete msghandler; + // MSGRecvThread_->interrupt(); + // delete MSGRecvThread_; +#endif + + +#ifdef GV_Scheduler +#ifdef GV_Monitor + bool isClosed = sche->ready_to_del(); + if(isClosed){ + delete sche; + } +#ifndef GV_Monitor + delete sche; +#endif +#endif + +#endif + // delete[] blockList_; + + + __cudaUnregisterFatBinaryHandle(); + + tool::Logging(LOG_INFO, myName_, "close the Server endpoint object.\n"); + // printf("zwx mapFatBinary=%p, mapFatBinHandle2CuModule=%p\n", _cuInfoMap.mapFatBinary, _cuInfoMap.mapFatBinHandle2CuModule); + free(myName_); + free(clientIP_); + free(clientPort_); +} + +void ServerEndpoint::SendResponse(RequestIOV* reqBuffer, const ucp_ep_h* ep, ucs_memory_type_t memType) { + SendData(reqBuffer->GetIOVs(), reqBuffer->GetNum(), + reqBuffer->GetHeaders(), reqBuffer->GetHeaderSize(), + (reqBuffer->GetThreadID() << RECV_AM_SHIFT_BIT) + reqBuffer->GetRequestType(), + &_dataWorker, (ucp_ep_h*)ep, false, memType); +} + +void ServerEndpoint::SendStatus(int status) { + RequestIOV resBuf; + resBuf.PushRequestType(SERVER_STATUS); + resBuf.Push(status); + int dummy = 0; + resBuf.PushThreadID(dummy); + SendData(resBuf.GetIOVs(), resBuf.GetNum(), + resBuf.GetHeaders(), resBuf.GetHeaderSize(), + resBuf.GetRequestType(), &_dataWorker, &_clientEp); +} + +void ServerEndpoint::UpdateStream(cudaStream_t stream, bool isCommStream) { + if (isCommStream) { + commStream_ = stream; + } else { + defaultStream_ = stream; + } +} \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/Design.png b/GPU-Virtual-Service/gpu-remoting/src/storage/Design.png new file mode 100644 index 0000000000000000000000000000000000000000..af3d11acf2d244f3155776e2ab5eee9b8b37c754 GIT binary patch literal 167076 zcmeFZXH-*b7cLr9KtaR;qEyQkDN>|M3y6r)5fG#+(z`Ti3B|5}iU@=f1PM(*dM5%Z z3WVMR1Vj>AXdyuQS#f_`_jkq_v3`LbAUh~ zuItyX+yH^twLqYKXZEuJ|08#LRRs8lCGdvcWl(94;39Cd&rRD%8w4s(c0Wp{T_hl-(KYGtUHz2VS`?Oeo-|-yWll}V!#KFd5`sa={4kY&bE;?2l z#Pj=3tMwF!Y0bZ{Y^THj|HlEQ_ZeUux=yiArfD?4VfuL%jp^@-mr9p(L62-nxYr{+WL zTz4phe2kI|-NUb0!2^oRxN*l4<*!(-vYr;PuF%-M z+a21oaA&75dx7x*g7KzzJ#Kyd;PtknJYPoO`;(MR=h#cZZ=lLDyMgQCvlGu7^w`An z85FpW!=>|rfltx`+rPKwKZ0yypclQAEcLJ@H9}z{NA~GxP#Cr)PW=Cv6r_8twVeVb z@)3Ggd%O7@l|HD|dadEGqC&vkPnmSr=MR@n)zeKP zmhlE;_)xhvyWnmu%-UmO*L!~axp>MF^au%#rU7h8|%VHs;w5>#c7I z^cO$o?`tV0HJQqe`L)$t659>y((xOsa=YwdWn}iRuom+)(GU^(W^I_i-~;>8DZ=<|{S)#Uh5p@x{T5@`bEL_^RgZ+}X|}9c%{tHwa}&YQM*J zoxaY{Mfp5JaM(=GX+vBo1e_js$E?U@DKKog^mz|pT)lk;$9N7xT&GHB%s0+}U~ws4 zey-IzC1MS1mJfYbJ$wFH6o^#gvHZBzA~yC>sQoZnFyeD-tDtlK8olEf>Reo6>FC=0 z*Ahx>p(-P2W7G&^%E-b9`M+u#K-G4c~T!b`umM$l|+X_q9Urf5lbuX2(0T)q$#qdXw$gk21r@`ZhG05=s(#) z>lKGGrvR^=;W@OAnh-=wc33Gh>r5S5ux9mxquR|d+Nam}S306m>BrNLb45R*^lYJZ zZ>sge^5+*P4CqsjG-@640WPZu#4b`3CUZx+p5WddbOH2o;F-x^^Ce z%ckC2?0w(t{oDL^&s(CbKbt&f(`t`$qx<2rvln*O#@&S-=rx)7^PKcG+t%a3Jk-JP z9rcFc?fS*-cSPQ)mX!o6{(VYOU361qud&&wdz&kxE1I;R#mWTULk}9FuILUZ_WG2a zmdG9##AOrx9M*An$;pXn`^Y8Otj|bG9?pY@sIYS40fvjWc z2v0kBwpaDcSke7z&trAuGn1Qr0@$gkMKx#3mnJN2LCb|pWwEh?3|en%##WUH>zQ`E zxti*Ti3sLxjgqGlY+2Cm)bBVxWjjkhE2(2t_J|aal-oGux4E5&);x&dzwxln-w?>EAVZv!kfBM%=PB*h{~S`~F@KjvS-(#m9n0vb%a=vG&Kf6c5N! zx2(X&r6&m;SE$;YprMfZEX7e{$03g6o+?8zoj>H2^@J4d(iPT+$$Etn3zr~HmIL!8 zxe7`*4XUkI7=G2Ui>^DkT1~lS2aG^uk%{HK-rdH88hyW=+(Q@llHZJR66bXNU@|Qq z4vgsgFK-|8En4D6esp%DM8ILskx@bG;@U=^AB`TfJgHe-)l70tY{zf<6TPQQosoer zGD{1Mu0C!*^wCeJ=@`c(Wcl-<5E1>PM;ka1H(c$|r3@0vSP64z44HW{GCYON&;Iwr zK4@-U9%R#2jk<~Y!mUYe`x=rJPLGmj6xMpbxP0-oncI2K6ZC3wAffXux%x~s9*c~0 zPw@~bR=$}AWgz{AdnjVoXSFHycuI4u$$?pwC(sMfqvHuFk8*Upu}VHzS-R91W2ykEL~HSVW5neNhkpf!JAu-ifhEJ(9xce_@m zrNWst7LFN{eaWvmdR9%y{pJU>rA*7u&25>rs5^5e`i+g6Ce&>Wfm4nYmWZP13@ zWrW|_As~NY9QKduXI`599KL)dv!j|XIPt@c$)h;`QOoAV`+WbH?B~x4HHUKEHo12k z4xX;VPoEXv{Q)FeQOWE}vntnit$UiEV-%2K0ii&JUKB|Ij3DaS+HiRLhu3`9u|EXl0ZWW1`HF2K&Qty{v(Zc|skM#Wc~l@NJVw)UpWEeySO zDoQv*t@rE+G)47pR6|eghx~(LqwS2EX6_3Fe!HUb_MM6@0tP%`;PrP>#ul;t z?3_kQMrhDgv?Mow;M~|T_YHZy5@C=A<)?A0g^s1j2oyc@azfy0|7yW$L(A8Li$S|A zuVQp8#e4Fx>W`g0{m2D}f`4ZKxjgH3u;W0kw!gfgcH?)x5N^$L?r8MaoeISh0=cwe zEN+T%|BmeWtgi}EZu_|%0{;Z5-4y(hD$SlQj~wqG32BOSS00@GxHnd6so-tS58fzal}WbULt{|z zAia-<-{$WDq7OScsEXDEQ?|TCyodW*5|LH3ga&+-(Ga;vs|LVB#O<{X^Nd6=ZD3`R zcuP*|S)-kQ&nsfjX$h?S`kSMHw7m0Uok5NeOXuF}Xn5r1h;(-FiD0|CnkxF0HiE-7 z(lU0MMZJAT+=OqBnnSaEF_@Krx<%-XpgiqN^vs}IdR%PqRCRwgEo-E>S)6iJkH8Qb zK_R#=F6>pX#H8J$BTKmv70OF1XT2*cj1l;&xC!B)B7%E7(0u*p(weJ?w}0p3P89VY zH2XP`9}O=eDJhBh-|x@PjV-aUI{n9HpuU@*RY65XUs?dviV4}B7|XUvGNVP*Ji4s% z8*PBHOP&OSs%%HeELznMsUlq=`{4ZHlrs3l{BtP5ST=vY$&nT&Yw2jK`tR2~4+d>D zx}8^zA!@WN*%N3HO0Y(Vna8R<6(OWhLcds+RsCHWx|*!fI)Tciz2j1=p&@&baKog5 zokp&w3ZIFB|2z;V`ZCWl$Q?5+)O{)c%|7^2QzJ#5K>aE~0I(-odyomQKUR^V8vWb? z*JFs4tAo8reZ=)e^v(uf2*j@Ux1(l{WdiL2Y2>L;_kj+{{~_oQRiS zUENa-PsGvDKb918?GO(5SWgi#Q2@%u_#xPcdX-N(cl46zAQy8B46jqlSOd+~8oIPy zyI(rTnxJrB%g?_*7+Vyw5HOu`wiNgK5kNe!5Je#2AKw7%m�Z#-Xe0iiEgV9pj6J zg?^~cOp)_Drzd(}Wge=sd~L2fuiUf|L&ETdvcq4HDDn+sSdZKXgDOpqqg539=JHOZ z7)g%EL#ow(;n6Ho0f&Bnkk&=djX!V%1X4&DQ;y;M{bNwtng6f*(*9D!O9jx+U_;S;zNj|Ex~76doTO1 z905X~Ug``aU7&#zFys~8ll{-KK)S1VB*fE!97wZ?fkLmQIC1z)Wt$ts7Cr#`Qu-VS zmv#Si^V#2|3h&FL>Z<@79p*>h29o_n9Ai0L%}#-Ku5wp-ZJ$BkBmf*M#XT6+GfCyN z7|lvwtM$V5f42wsWFC1!vnJ=(og|<|+cc?E{e3ebJpNptRmiRjFy9HAQH8%}B&0hHef!}9xlh7n3rdtx%7@i@kA^Q6l5zQzQ#j?P+ejnNLdgO~Vfq9o+RNLex7 zQC=v84upbofro$#(0rsW(`T1~EdH*;Hp!}ZXDIr8lfdt`wE7)tgbj8Nbc zbEF_|RD^R!hm!2>hMOa7Fd}sS+mZ3AU^t29HQbv2Iluu!e?^*Sv?Qtfg$n2PXb!Ml z$nT*fg;n2!bFDv0l$NvQj zw`eY$-?bN4&t@8sY_v(F+oQmCG%mJqAFBoD zkw5fS^qIVggxi<}h_$RGJob6oK5$f~TBKB}X9n^YY$C$uQA3OS1BC%vRP zeTU6FIIGC$`I_zd5Ls^HMdvJfKwABF&T7rcDKR4y`>?msIl;*u(LVIse}~46@u8tf zf;Mdicij=JezI@Se{f#IhmpL*ImEg#G>WhlM$pes$#Fp#a8Ffb!{y!#|2y}hi1R0T z`)qG)IIN3nu%6<5GSu>B`v3!XUyV_hEn<`j@98L+dnBnR=+)a_OOg>`Ge5v_fzo=X z1iAyxN?pk%H117MXr}~M4<>Xa29dG}!(ok8V&FH%@B{(Nvz7puxF&0P-afv+=Gnh8 zOvUzU7Fkgo2?Trnsz{43gp@nD#JW+DH)CJtniCtRoS@OVMI_e_U8Iv<3j?0J6jybT zzWX?u=Yr;jk`(**e;VbV$~~`w$cL#gLh`2hNAC3w&4^ zQEZSw1sww4WjIMFDmoWys`%}WFJmb?-U>jXB>!9o+TK>1ex6jKmrD_*KY|aLwgR_6 z^#FM?B{QUDJQs2*6jw@C^yc;79-QGeGh*YD|5vj9_5>DhsU`PpPf%i6_$V8veRf&N zjgOmHR;ZEzlwpNN=I==zZ37Nc^2Yy%jY z*+9vC|MG!n769}D{=0BbN+>O7faw5OHwKMQKrF~YjgTow{ua5|EAp4uQ;Cnb*oeI* z;^oe8@(+u_D&Hs}0{AUn7916sPFqp?)m=aU%B?m@(KI+RZ?4=8Qw}#jz-|U**aVNz za<0zD^;qGAO6`l6-!13Rtz=;I8?v?U{2SV|M=t-*up&1*&67_Rqgo!v1ZW2}htt$4 zifBO}#E^QaW8=lh3|Ymz;)(aK@#TWUbPJR|JT{K|!nkmxzUi?VGewbKeu9`W53wj7 zG<&)Eix|u)ypGm6G}$W%kQWG(-gXrCRbjphh4M5E!+Y>oH#EF2=5-x-{XRb zHzWcsJp@0IcI52UeSsX$)sQ`z0^0|y3%(`@dG&^ke7Mz`(JcUM$6fk%v(_}J zQZ29mbrRv^TIVsE6fv~6S3VQSTe}~;&U;4rjPmFQ)|K_xiKUKPn>TT=NuDSajpV&x ziJ$O4R+}WVk~L}r8y0~aF+3Nv%s)l-wblELQ~FkTGH$$-nCdIGDs#d=aOrwG9&N4i zu{v~Z+Np??r>_%u4`*d*>C*8fyP-ifwbS9^ym_Fw#&(~Rgmz4Q0oF#I>uqMB{`Csc ze80)igz{=(`FDu0;WBr`vgFu(^sd7zBeJVO6htY+PRh?a(>w0DC57&a1ru)qLC8ECYFziWrkAc7B-Fh;de5YC-af|rAm2njA_7N@|=*zhGz4)zM*ydgN(ACK% z+~Y3X=H@+lm(Dj%zOEJ#))EpD-JfFi;uR|RR^jH=D(0+mud+OC9Cil5Fd$J$q$Vv` zS)#_ts6EB?wyfvcV{Wiomr)*BlazGFkHRXCX(g^5u!yWSj z{zkjy*>nfk0PgR0;aeuG)zZ8u0d~idOZR6-4yI}F6qs+cr;!4dj59gVz?P4DCo(9j zSiH>c5}Z+hNaSSwg&sb)?I0OZ<~MHK$%@?k?!;_OqVFh&tXQL)3SkdWBUBQ>2nnIB zZ1aZ`mt4~QP&L>)MGEc=SXor(k1x-cg-p=Ro1Ky&w#V4#_^`*LaNuL; znuD4{t#2mRp#%Zt9^u}wmGTZlOiR?+IB)6rC0u1iMd@}58oSXU!i`mUgD2Ss4_j`# zm2=8Jq4`GiUMW;K@6QxlvEQu8#3f)cX11yx^+rc&m<}E?u0m zd*3O6mxj6|n1ml2w{S-AVGHe9v)rrW_+(}M=Xt0DSf6#HZ|7dyecvZWAuQ zrj18w3LdWui;Cp<93ulhh72xR7ja#3?LHBqPUN#OGkbNAXF7P)K4c|g;oalctpi~M zY?^&z$TEwyX zpwlSmUj9aBufBl09SK5+s;Z%^=dl&O^#l3puhG7ZsDxaxcva-)=H}g&I%@a!MB&>H zv27DKDW)H7JjN?ro})s|W?*!~63u{=PT8h-o>S<^%dkWJ3^`ru{X=gGLw>?KN8+c;%Dm~8}oe|@g?7b)@Z zeN)rqThbsG0gK`sF+QOgS(jPn6Xok@vx@$BTj!o*a&mI(q-tXbzDQhLyu2$#+Md7L zsH|MUJypR2sAxSoy3b#CwcGLM)R^;kb#|Vq_30$lM|ob5rsd`3g)|!0Q}^SdM1ed3 zjBwf}A<2*uJ}k~{cXE=t9wIGM<--%&4Va8I5X8grrxbC-F2nEbuFrnwf7&8#-|&~0 ztnWnC_k?tP#f!M^v}2J)a|vf5@^!GAMpx@Mz6#*S@)aEJeLvTlPlnxtaBJ>n3l)dt zk#SA9tcVrdLr%U4{Ey4k*6fD_Pec~wYN{E;+El06se$3uK0J!ck}@s3IYN8I1pnX! zsUieI^j@k;z@3j*eY2vrZ>c#nhLqq_Wi*HB@EuGbTyK7S(;A- zU$Fjo%6hNXzj|dcF^<*3n!7_Yo~#JYD#d&}x|ZF%zKck`nkWqTe&kvNh7+;rwfXZq z!VlBpx-?XXE$78e0uSnBl z-J(*tV#hJ=}SzK}r3YHcY`@9rU zC*3lR7`lAzO)0p&@#9M?48-wIhU|sX11X5jhkOnvx|#i4^h({QvDK+Yyh?(xCh4b> z1O@iO=n|4y!Y5KW41LEwwkBAfHD~FHychL+I%v>HX@9zpJaVyM1abH1mDRy~6ItFh z3X2xuPpT5zkIM*55R&|u;^SYQmk~5@z4gsS6ZMTNvX**LZVjjISeN4RK{L#XoDWnn z56WK4=o>?ZK1vjRxx3e7c~Oe3%j~Bz0mnF%s&9lVRo_qW@+e~ScJ5IpbUSsXCQDke znS-AF{Y1bcfV8wvT)OvOTvD=f(E00&rw88`iv9%lFwR7(%pgVDb>RK=5S!Kx04||D z%wGDf&$NF3wT95;lG=$VPuo$2)i>hih8`Y;nK$8?<2QT^LnGtur3_M9C~HWh1Ii$; zi}m)Pnd&gKz!TAcPnMC6=YU8=qfg9NdTm;vmv(RK*S^)AKO0>W{|k3SznQ4^>iG2Q zdWi!}6TaTc?&D-q{TJ`G<} z(dFCYjiG+&r4G&E<3Yizlb22g8R_Cmo6P*{yhcmEN18cV5SYREVZ2Vjx2~1Vu(~9n zEU2#OhRqw_c~najB~}WE=|3sh6-V|7ia{C%Ee(mM7RlI)yetyh1=3L}04Qt)%wjIH z&MAnUK-k1EGtNYO*Ez1+fb(-Bj#{W~#4AsKRMfVIxPR!?U+a6PlSF6(o@TiWIh~*g zo%s|KGSTm5B|z)|)60dFuSSIfIYZ&rpt*&>up++Lriho9HzaFMqJD~4vwm&tOEWvo z;>02UX)*VrUYN*mzI#!GKB*~^bg$CA|4h1aCtr${FI?YC#Sv5$h4=Ji9R%IE;-|y{ zs?aG%_!O6x0=0~ruTRx3?ZFx_HZkg0lK?!XE1Qiept&=f^IW>Z4^Q8&MRH-8z64z3 zxol#G&6{{RKxuP+uD4I9-~Y%3DYdEY%m|$hYqhIVYHYEGj%FZtL8yETwta^a6iuBC`_Gr!Rrr$aN6nAe31&x7O7T-)G z^r*?CJ|FqI!|5JnTkET+7P?wBxR?DCu3i|dciPh0cv+ce!TKdJ1Sktr=MelcQurF>A# zj#>%~v8oQjJ~8!K(XL8RdZfI2U-p|g;U_yw!(H7l#jcSCooybck*;H%0E}VH;H3sF zdzfZ*WO71k{+dtXgmf58oy-R{>B~&X)YO83ETYXWIQM?Rve)?Vn-60U8&&6des(@yKA~{S`XZ9IGo)$CmN&GE7{dp-4 z{D$ZDbcEk#0Znkw1P(trwt5|O2tem=g+JEM=niGmpNrztbYsr!fUm@!&7qE1%iBpyXBV2Yp9zpd;4-T z{~&1QEQsUNM3slREUyubAy)8C zxs><*T*=SL#oGfw{t}KXTH{0R>M|8Jvu#dAni>a*l)Fnzyzo#K%+1@kGLUDmE>4Ro zTizOTP8>&|EjW|d`xfH`FUj4S(+S)8e9UCb1El3T)Kn0w2h^$qSOxmdkRp50X+UJ- zD}<@>LBB@lNesGFtjQwNrVOFVQ)K>}x+pSbunH|YST)}XKF_sSr_PBhL@QZ3Jg%oH zbkx^ZxnFZb!4v`0u*$J6R9po5y>LbCDNcKOwLyx=Y_I}t!mDiDLk~U`o;K+>8|B(& z|J<;~Yl$eL*aHup;41+TA2f!ocE)fhF7K4sWm>RD}OCGK*eKK&2~}O{>n{3mVGy5rno3zZQP&MP4 zGGMx*(F;kQe<_UcnJNVx?C_&kmaJnI62cqRCO-^6muWd+INTos(Nqa{R_`vWx3;nI z^oq}Z=TEp@l$V+S#XMdM8?Hm?8NK+rnq0F&CG>J57LXkEx2ippX;EevU#;>1KeB(B zxE=Bfd;Lr-r#B}nx%7d@^ox`7-PNNG0^QfU4f22SFAz))_UEvpheZ+|#%GL#7)0!> z`q&;n5ZGbrb++^>9Nr?f*wb5fdVzZu_Tcs)!l-ibxkSp-&1==ffnimPcfY-Kl29`v15p`x7^e&9J$`= zB+(zv78>i&W7}mvj0Q?Idwp zWZoF?&g@ixk?a%Gcrw`jSb6$>`x~Yc9pkRH$%THd>GT_spvsz3*^M-j2jp<$Zg4Kd z(i$I8JoU#ecar?O2s$F&mqYX16?^5OlPdmBvdLNuOVG>oAeQss?3{aP5KJ2~fTOQI z#gwO;Q)aZw+_&+$bc0xGI;~~?7uG&%l-J-xO?eU-9@7I(FnInop)-|RW9wz6Fc=aU zMP5;~0a$KZpAqd7HQx-}&H zgT0~SH?qg)W9pzf2ZdCt95m_VQ6w-%4DutPqjnlG)$^QreF z6AYR};R+>Tv!d4w%oU`tY{;j5ot^sWJ+tqyRaM+2DMi;EeDbl)RA&Ir<#}&*DMsW4 zPm=YaLl<*k#|Z2~gB$6Dfy2IUQJ55Xs47Ob)-Vop*Izp9XMUGGe4VsB@o-SmXne#B zmaN1eyQ>ZpH`28SLu@*F)xcnCq(wuULa)^5;kQNKFM&(+-FQ$tqZ@-K6nA&@EgBSh z!KIsu!x}PXJLPOs0x1UgRn8%2Aj58d%CLS;KMd+9oeS_ASSb)SAzPGi4+7c-O zbhT9m*XUt7o|xjnFuONi3w#hI8*`WudaH{@Y+qc5N)iQD2~fwJ&Z|f1Q$0cZ^L5}v zbcx(E6VeaH@yE4+9cg5izHMmNqBwxE@ zqsxh(O8EpXWn(}HU2kV2PGvk!dvBnF7(Lx&xVSZlh}`Iv%+Q3|a-|86;>zay65D>1 zUNjYZwwQ&b2te)5f;8#;po_x;;IVMi*@W@20GiXaT@gB6=bc3yWBr_GJ-CkDE*!*L zt*-t+aHvOTO-|TiY2w?($SC^Wzl3BW4O7ZysZTg2NQu2S+w%<8JwG5 zGs_o6jass6r}`uxIv?3=Ki-3r;m%hcuCODhWh9aXzc`+s$|h_-+*wOGLQ~(|+xps| zp`!i}1mf|eAu$+AS>D|#0E5%#$keF2n;N?4pZ}9x0j%~?5&3=CpJ1IVPS)Qn1PhM3 z8Q&_nR46-IJKvM3hOfIg>IjXJO_mMBcE3npzX6kO+5K^~_x5C6z;^_uM`iAynsMHc zmxCcKnhbXgU(9oo2=E@YxOx5h)B3Fe1AJAk>*owGb(Xu;;tT@@)6*pGogX-&i;B%# zS`$P2441vyl$#><%h@Mc1T|T79K`88J_L*`x@ns!NFdT1Lf3pd;)#xVz4WknVGY&d zh;tY0Tb3w42b;qHoT3npaeH062%|QWu<2*l9wrE>v6-jQGywI)&y*u=TeYF&twn@Y znF9i7;)t*iV&2lRiQ&3LaLJM^Nj0e?7@N zi!$`S#Bnuj^ZR+cN)ZW-=)|h-k`yC%*S`_}rU*`3X1N5~F#!V#IEN7ZL6;UcZMN>) zoqExAaUjoiwA4O)zVpy8it5@J&=Lrh+1B<*?gz0Ymvn2EZXj8kqRf}zUvxMw%bNp0xKM37Z3I$x0*0CN21molMD$r(O!L_=>Ch7 zJ7c5yCU7T-*0uA8++V~VSlq**RlW`_rXkVfs2el-GD3@f_ciAm)?S^}o4Dt=t1Fz` z$|Gw?RLjL2p*?!|SjE)IG-mQ^ry-E5ro2i|-3r9jmLfY9`7InwN!mNFIycR5TV)J# zS*wLTvH}9#m7pw4Gc|b8c0iGQX%*gR$K9~?sSP@IG2q8%po9u$c)r*ZJ5ei~vrrPC z*HO1p(FI5(A)mfLPwXVYsh>wwdGIC_n4HTq+69BR7Y-nf#)wM=;QC-1hS5nlyfw2k?11E+PG3RH~MiSu7)F$vQxhL%3tb5P*|~671|+A_!))z zsp(j$N4f{MI{6Toa>>gLMxK5VwrlvPCo?luSQkJKBd)dLKGG$6v&X5l3lVNJe^Mk zu%2gUG__3{xby%$M3SQR7a6zeM;0=OD1heydM;oS>VQ7tloz_jhZB*r*!pTZBNX+h zzLoEgvCCx^Q4Q1ny+;&L*lhuzo^sbM+_-NzgE zN`m{5P++p%I-E^NpS)+}Rj*Z0SGCBJ16^T4rBwZI=Mvj^{vs^SCW~A%wo;QE^l3Nx zv6#hytA%C+v7iaPU~sCZCgkPt0d^vB#<&fIc_gwveLnzb^P|m41=E4%pPnmP8 zW(I``_+q7|;tWArgM}ZC_`cmLDlX+fET!;SAo|vt9l{6thMaDfQnsObJo|^AThB%p zN`@XhDfc-cFA&#*?bj1AVnb223B-YT#t$=tab_2loQ;8|aPz}-5`{GYfz$e0*lr-= z9%^l!2mu)W*g~pVS{rm64=_Oi%_(!6{ZyYCD)mN~ngpi31oD%RwazFxJ$qA;nQHDn zW~Q3Lahvm#lWD>e@%)ONEtlZEuI+Y}-bK3EkI$yDog((>w@Si+6od-jS)H2Ej?0bC z^#qaQ90g8U%|>SzN8#8Vr{dmJ_NaCx2x&YBqY+2rOrDBvP}$?^v1Xz?hA2x>w1i(f zY06IVlw2OwQtJ48F072p+;pIfwZeb)vf8#6!86Q@jVo|*hsWJ$^~+=9Ox5v=rr)+2 zSJ|4I-pjpl7}|8qF*_+by!HgfIUF+KAQSS?@fn9XmzG;=EbI0|A>{`ym43-ylU**W zkvTT|4>MQ=X)Q&QKN50V=i@rvXE)Osq1rBh<^^?K#-+YWyqR!{+QXNeE z&JTGdWnPaBRawt4fL5ffQ!IA6N*eNWU`WeQZq4c^vpr`m(=-LlMq8&w$yp0|An-MT zQv9UFEd7dj2b1wN(gE>k-P{6wg09agcOlsK&0>{@y;B8XKY%XLNZ5*dee2~{d~Nw- zZVrdVch71z;u6C@71+F;g=MF?>l54{S>mZ5)Pt$)G8`XC+>%7ue(q8dTM8m2YI|0W@kTs+Z!@W~g`$Nvl{j zkZI;IoKM;F%nUc5Kh7sDupIuAF!6w3K>D%j6=g-0%Nhv4WrniBH_zG))+oZFnTNkP z4geI!N%QP=U}ywrjCZ399#Q&&qP!d4iBs7REj>5m-Zuqlr`cXLJ8Xa!JAq?&COWk% z>MVm7%+~w3A3bD89`~-HLbQmdmKO8jhct)%!VRd8lb6_s)*=)px4g^=-wq(P%RnWJ z8F_mxSs;qJgNEF!Ei@ZXQ$Z@nfcYc9tPUs=Va5HqdhF&08vLjPlpdjWd_Qiv3`$!D z^w=#Obe*O&?&TQXWXmB1k5rf=H*ns>CuNnjB;Bv&lHAx&NO~ZE z;_lr_eM?)iLIQn}`ueB6;(P5h!KWn2sCa#yhB@42ZCT`p6K^FFVfv^(M5puo=-EW| zqmiA)rKA#@=7fmW(o0vG--j;drLdWEEn@i0+U8sATwXgi7%#y!Fh&ZLdH1Q&lKy>6 zU7YDPKmp_aW{;kyjXnxa;?~*%iCH3*ITRYAlK@4oa+7?X7CcsgKPUEPIOo*xghimm zZ1x8eR9VrYEF-2Znq@B&s5s?L*I32h3>ECxR4}1fK;GxL&QSxoIoZEVrqhlZ+t!Jg zf*FIvOK*&XA13rK2?EfiC(oi((zFi5MsZ|!oCHu4-t z!{Wni0C2sJ3NVc6*L{9`{L?G61G?yiKLE`}(Pbtv8YQ_Y!9dQ&+@FhY%O3kvkZ%}E9SRv%uzXNPx-x{i>-2mdm^-I4J-i{fXx4g;_ zSrw3*3J65`A)E5&A!%bCE0OZ=G<|8Gn)X0vBfjQC&BaWp&dY+-&0yz6&lT18oibB< z$q7rk-wG6z=IeeS7f$#tGI>y~L!O||uUoRDNW3L9cF1OG+QD<@&M5*DMvfZeR3#sq zpABYl2{-}G$ooITVKM;q^QtTN(X(e9jqz8ccGwx}0Ii9~6%gBWRA`HvGWtb?1GvRU zTmolC0|5b~Vq!)8HZXZ?y?MZBJOlubua0Rb?kU?@|bt>%j?tRuH>GN*ONrgr93VR3%u8H1wE3h#~mSri6o$$ z4*iH;<1>wv6W5#9*;OI$T)D`Bi|Y&&vs+c&`!FYU)uRQu(V@6!24MW7KP;BaGiimE z)A+{OM*?TkZw3xXQD(WJ04+0GXntO;1pw45D#;cU<50tXTk5L1g}D)20DZL$8v<3lvqtnCmfz3scS&p}21v%L4A$2Pi)AO+W zPX#dO=HAN_H&ME%j7##VgX{OndMzUxB315WKo(Ou=(%dyaWl|G+Vp+7rdJE6m}i#A zg@~oXeE#=(gRp#jn-O8D(o0xcu&Lk8$I8sC-$_m5uN#h9`iZ%7ySNqe%G_1;j9=t% zkqIq~T!rz+B)#?mLXiwiTu0H5du&OsORvj)53wcU3#kVk&j#$+0S?pbYJslxbM#}7 zGoFdKLpHwf0jHWEr_>^;-iE2Nr&IM!v-G0${z{6-D-R;!1e4hdH!YW!>O9}dE=6Qq zpKh2x3OS;Mnp?C{gBH6_<17bls;L%;4T|VTn1?KSd*uk?bMmzz#o31eUZBmmM_;? zdZ(Ks(0t~9Vdm5Q>$PQC7SEO5$<>?y_-uhl&+*M;Htfh^F@-NdCInhMxbz4}M3G6> zOHLjv!8BQG{->9`p$N5IfY6p5fzFSAP`zm{C>uXu;JhS!x#XVv7%*V?_!o4tW!Up! z!=Ttz;H?d??M-0>F)*{}hg}B5TBaYjK-amGGO!Cs{pzz=KrR-kkD>a!vu(N$NDmsW zy8*}3Lkhm;!%(BL%&x%~Qownz)(`1Ozmw~G0VbKqlKrw!S7LbE{JEjeD?07Nme&F> z-I1-;%uIhvVT`bW?j`^e(CqAZ5wK>eXBNFU>;8|Yz_1#a8y-YFdrsHJ!oHb<-d zejOKj**Q{T>+{L)^OGa38Ex+gg1|t=$HBp<+85yRdrbHaqTX^I_UhYc6E(^5PGQkH z4WMd}ZYe-k0$Vsf;H}_-ewzd1giJWxpMB^g0lryc!)tgB79cr-^;7npJOR*7GiVJwJ>SPkx!VZP)uewJIqg~)EG<)vW**Kbf^L*j{cGD>ph7q zdH$+rYM2ehN31OIjmYtg*`xK2x$eI&El&6Ee3~+iD{^W#lWIYx-hYIlHkstSgY3(8 zm~7uC9N2nIDWL5a>Mle}OSghkgonSq85p=hxg}-rl+5UU&oIx-tsy+NXb4_*^@`CX zbjuSrMR`*aefQ^g2-(i9Hzf45qYRRFCd!Z{GgUhWH#}z_^2#6ITY@>H0o0v z*+(?w1s23?Q-N%C!NLld4*rrq00zlr*S+E$fUTNp?H~)oKB83N9+Ky#cFL`qt!>^AD6^V6jezqECy20^?~_9 zd$lzG=*z3ZAc3_h_93@`GZ8$KL`v__Vcz!#&7};6_XI~JT-~6o4P_GvMjph(ciOmp zAkx)eg39u2gr5ifIMW=ZLy7M_PdvkjxS!sjFt*r{X)|J_-zRK+>%Szl^E}~!!p*Av z()C9>z>mOn*g%$3BEKH$)bZ&^f)Owl)jj;H8_nZ?YF2%i20150lqn46leV)i1NU8? zjg9No*vSyWU9Q$jmRa%}#KA^Uc~2+33l{GP!CN$0s`mYn1B?Y!^IBF80@8&maoGiX zJeMfuG>C-h!VbI5wN@ooS>C3D%Fe~=*GH>La&59pAARU0PA#%q*=t>b+er=GZhtgh zo(uXAI|X`a@<+E*p6leQtNW0PTu(n^`P`HcDXq1%!?J{+&3M@6gK|7;A5mB=1CbnL z*E220QfzHo20rKKeTn1TRufSQ_JT23Sj=!s-!$$#Z=7SmIRXS3{-WGjb@%=Uh8 z$--W!UfD8m<(L+ABQc{*3#5=WF|qS}lNEHTN31rdk7rN=wWCb0y9$a_h~dm077U#X z*uGsGv^G7JkX}?CwGE7+k3Tvt;~>b??mT<|ypVYTS_VL`P(Gw?*K#9buh<1D>cYVx zQ2hB8$M{@eVc&TE7sch*3xU6WeKCfOC7>e0qi+H7-9y;Fy5V3<4_rO0g7!glo=jYJ z!+D%g3;BRgl++*6oC@gyhSW)fDu~xTToINKvIZQE1C~*r6RA!5mDNornNZ)-6D!*Z zR6YFtsfJ)zAj7)?jH@e+Obp3V-yU;j%AvS~y@59bfSfZRVik^*V~Xt{BeLFOADw)c ztt;+66|%Z{69&xR&>kSMtDf|m?)2kEx}?~@pE|n^$RLNk!o^6yoPUh{MR;_+5>-+{ z!W$fLZ`3|?Jiyta-I=sG-+JCz)G*Zv?A2E>64bBDH(o9b%K|h>A^f0culB(Af%}h& z0eTMLG93dJ{-hbelH-q`0MNb)4oJ}4vE6CT`NZ{#l5~}ToT&8%m#*W3c?Ky?(jXp@ z;g1HEf!yB!4a%=K;E@Ib=1$3n4;M(X1EWmhrt^xDIXejSl4JgFWh;jO&_|ZPE&3W; zTmtOiOR0Zesq{P4fOq;|%1G|KYiOAGEe-*ajHw0FY=$hLABgwwKOfSv)%n~0(f5I~ zT-jn$)*nBuOgE#A%N&{~5^TGe;#UB*8v?hlJ%Ek!;r`E>e#8L8N{$?adS9*d!v_Tb z{Zw}5Ap(BujXanQ60d; z<8r?mSlFa{nL6GEMU#P<^Idix;E~?Q{MtQtu3wqyFeeb&zhyDM(!qc&watnL#NUB2 z%qFeV_JB@7ckOlGi6bIlYkt&#c+hSE%RdDkh{pe0>CNA2b(_pRaGd~>A@pC4yoOR z%2$0T2EOFv6}F&3+bOP}^!U z%n?=bLqi^dy&Bljo?KJoZmR^SyD4hqPRX);flC)0n#Q((6e+znSzmdp#{1UC_osa@ zWscy+(no-bbDTt$8u6vV)=#^}t~s*V!ub?h>p2!Ps}lHI=pN#sVl797R!(qBAN*K!ku0X`_fr zmjF_OA}us&L68sxm9c@17OEg6^cFfqM5WgdN&-d&f|Srg4}rTjGxMEq?sv|)H-9SxYNON>KE~|BFlj`uLqu&YG5Tg7&46NT&xIIGeYXPsqRKHR1U1+-l zM^-uz?CRnbngI9BQ`fx)wo5k%I`7R#d8F&bIrP!1{PL~S>9;wj*>$RZXZ@zEDn7j1 zQ>u7oG%P%F{N+9%Zi|0{?Avm+1_uHQZL-#9PVS}^`uqqP=(s%^Al>d>n1(TSr71ey zYpemY*7!RARA};#K3?k^2$`J*>YNhcj#&8y-PrS=2eHdT5-zT_fGjC(vgf)+OteG| zdK9wV+B00YKh~oFUPc}eJprNkxFcNt^(|YAPaLx3lw_8Z7L#=J9tR2nttB(V7HsQ4 zlPoSWv1z=cXe=ZiR0I!nDHGL8wdz#C#OU>(_TJXag#?hkX>?sCODbu~=Z26!dH{E{D!2ohl*g%eO?!<|{4C(CIRO@nH*|-Fh;~ z2ev9Jjcns@#_%~%;ehu3BmW4}mdFidm45Um9RsbXL zLkKc%woe=1mO1W&tPxDMNKg0Nb82Px*4GmQ-%g>xb>KW1tP~o5&f}&xXX0`#%eZxf zZqRR_E%E@mjlsUCr)5S+N#AG0I!-w*5=NKunF+c}qnXF7``+hSDrG%EEY751!RnK$ z$m?arefQdiKl8rUnBPTJj z+NRakvWxi?GuG!`7s>(aOAVqt4Njxb4A-M3SVq6eW$nD z)V|NmaNz-NBmJ0Rm7aIYmn_^d67};85Ez0Z@g9|uX?+G( zm>Qq?(OzJnCAw;}~5rl!3*Z$t{Lz1!fWNjY+MUt$1{iFZuG zDps_CM&Z`-J03rm-W<6-N6r?o4N;bP6%_!+j7wC#?@%pk1Tz{x#MGID#MU`C3qL|S zu6BX4MD$_jB>Gx!*;IgvL0fia<~c}dIH@}K=}^-!fiLI8rN{l)y+_Y|$b0k5m1Vs8 zo~Kl$mcBPZgd2@Wm>jI|elw*G!u>X_k_#iYmSmr|B-#S`+DsjQl;jmsovG$ErStS_8wl-s3kHeJG22l*xyi_JDK+<(>M1dXzl zXr81O?m<~akHeLq9y$A=4~cDN?HkFd**}B2!O?e&!*DI~`$)^m<}V}ig1>*O3uP)w z@=3tTF-!}#IU7X^!q#9VJ`lVfw3}cPpC-|R2ATwX-BZ4e#X2TBvpN#OD!Dr4-aft? zFyvoeLI_M{`1)EC`33-3==Qkpny+ikH#m@v1&n@Z5XB8nE5*}|kl|KEcdpG!!11eG zoz#C;k`n}u9Zc-;!lLVV{!gHuLSMgreacfn+Je6|mq5tL$pO0QXS!s-{Gyv%T0#wc zfd_{RFq7~kiijB>>z0=_C0__wArupY6~_9=Zh-lckv7OWTU=C8GgYfQ87bmOL* zi1ljtmRViVeMkV*$0L^%IbP_R%phgTom_l2c0HWjc8D12Qnd8+Rs(dvQZ15VdwTa8 z=iW$j1ZQW{h8nS>_rMr^D!tK+IuwL_)=gpXQ@-^$MxOatGWAMZq>wz^5K*Srs0OO( zrI?Dh0>~@oW$xBBf^k38CNHMw@}iU1h{3!0{KK7_A&bgtC9&|rcFdJ0J z3Bu~DBX2PAPs35{8QK+4rUw6>>h&J2R;ig(Kd9m(3j|kS$9*{>qFWi9YQUG;v)58G zATuMq0r|2!`Shq)WK(UAYo5q~tN}PY?FdDH0Thz%4ZOcsVpl_5FrdI^?B1n#O5NDV zySn0@N5`1MrBNQnKr|*!Ym*(mN}2fzkMzK~ZY+H^s}qEER%T^qGY^=di%jc2-6h4R z;ugH#ut(!>A~sW~qZQoB#zrb{Wzy14vDtfF_K*hjUgej7W1bvNdFc8!bN=CTAhA4l zM;MMqhQiP?`~^L}^Bunm_vJn?s6#Fn^)cGwm=_LnS&#OSHhgRB(0A*)E^K1K4kxWM zv{yVPqv+K0V~VaZ*b{9vvuR#2lhJDQw`0`&ylt>g!l2f@HZd$=$!nT5O9ntC4VNL} zj_#yw3^Y5SBwGmv;*`goUaGUk-`jDSmCoeTh~6EVWwN-t#G{a0xnVU*CrB~+P=tEV z!sZoi?>iZxtxL=5aLCeM{8I_h_9a&oFH1^M+!@@P@$K<;r8Dt&e+Stn6=Gv}J(DJC zyiBA2HD)q3zIf9mF4t%V9JL0}|o6b+o!JQrCbx-kpV(td1}xAcyIW(+EzJVU9yB(Ha%*f9cob4(<2X`I{;ZX)%ODTUsp z#EO!w7;*Go{nE#$luE*uRE*CbYL!h-xD9d$JsnsmwIX7q%^I2K@lo;M+rJzhK}t8x z$t#`?jSqT540=h1n`euS?TcQ@?7Ne@{khWf@mq1xZUiw$l}+QhTD`LFfXT=j3%K;b z%CD$C{5zn3x0_x?UZej1{x*OXjLAkfmh$>EfJn zqUuJP-Y_p!d;vG26Em(Q8)=MGY)rF!cRR%^W?n)RS&BSnGj}SY+DXkU}cY~HJ!S1f4et|D9JwvU343AJ%>}oK#o*++>NT>y~$ao`A_gEL@$(2zN@`JDRECvv8XZJ zw~wpZ9`azsKfND3@YbS$_ng$Op?>ftP-RrLotqpb-0HyT9Rl#qqWBPkWWx>(OMo3~ zbORp;^Cp~U-C^8{TrzzkYFMPlxpwoV9yvmNt@-Pnf`%=hAZMmZWjHV4Tw6-Ozm&S%h1W za-iH6&WWr5sxMlhOK+cZ&eAB5z6!oK-)3%vL%yg0MKAaU=0vv`w6J(GzWvjaLv4^g zN*H{3-ba-}ciXnP;iRmf#^2AQ1esTB0FxgHiiZrpj} zk*ePYFS4?>#_N?^P$j+~FFj{it!i|&n>hS{(dvOIs_XE>U>$1T%L9x;L$jS!z5FC$ zV}&fcrxewGn_@*8AOs1eGOFnFvmW^_t0%JP)0v$2YYFX&OrwTFly69S-1Ur76Z59g z$UPVp)>ubb19G;@BhS=6Hdfw|>a4VXPX^tH*7eIZiTqhd4)_>@bm zndk*HPjG*i&`p3ew7Ip=MB{L4xDBT~LEq|Q)cj}8AVs$BU^4TYhXXA+NSU6!HrV?! zkc7$6NDoPxz{7T^M<2Gc0PCG7Q%wpYZ^kk~L$D+CE6 zf)}L&z+*m)Zo-f6t<0cYfA~1c^@mzG9aUqM*{iZ}GBzYS++wuznZMv=qUzzPo@Xw3 zvIYqeHWqvO1Wjy_vME{#85W+#`?_Z5euR(;VSznc%n8*{QY>hFAezCF6&Ks?gtuak zPGNimyl3ADx4z?HCK-|9cW!Y}$?fREPoSVgCH<8JBx#cL|Rl?BL2wU7~Lnd<6b!m->Sr1Ja=<@iST-xt^rZ` zaQWccQUj*ib26;SKZ|%wOS8S6Pwm5T%BqIY)WxC^4WY)K_0;S8PbQmR2HfFakV(mi z>QJ_stWd$TB4B!x$4vEqGiLoUyzB?uoe;3PuN@^F>jpZf{YAc$rpCZI0BZRyKh#A} z78ZTPuRUHF0HK!3LJCfXvxIjSMtN{{+g8Ahk^9W2Xu-S9kbnDz1OASNFMX8PZ0>T4yW3Gebs5&7Hv_KyM@g1T4guIQYAJ^kJAEKWWZ6e;&v zHL6LpRYG|%aw z4AGAhhx>DKgsoo6zd7x~5zBBXG9}{${D4~YTt-q*CgKgFR}B;ro$R|7T;Rmqx7%sRNj-1(%+surxm;NFWB)vQ8Fiq zof<~Feyl0UXEjlE*`r7e=u6N{Py~sZsH=zWghA5}gNe@`rz(Ee%q(Jh-iM<;RKb z=SQ3W05(mfj0B7Y$T#WTTV7FQrxR%>|4L^-C24dBf<7v|$MDU~&Ci#{@s^bk zQ}@aEg_>7U*OTTirEFYz7)|^PQ~SR-1!x8QSPxab{b?5X|hLmXiN?XS!>$D;Cd{R5 z^31y}U9$K+z$rmGRx5Ft)Af9#tA+8J`n{4mju6dLh{y|USg*whbNzmJ9W0}5?eFZZ z+eR-%$c^@`C#()E(iI};o|WbonEY)2A6yM_&$z*kRAh4C?YA?*Q7Or9Lws@}SGVGd zxv8l`i#_mM!OtWG7U_|JIKhs`%tb1R-^WBupGd;ffQb55L}XL4t*+%e)@z92_0&PP z-B&zx-O_uoELIR$kZYnv0)D=;)&E|#^5dCXu(7P+Pmd;$k`Q`oJQ|!=Fe9*#|HuWU zMgLra-&bd#A4OnXi(+$qz}J;*RQCOv)Vgv^Xp&1=pr;*Igz>-7=nvt>-;+}z(AI!k zIVn?EBr~BFAi3V33+#4#G0Y*l7bbR}w_AKRJ=oUe-gL#-)XEagqinPYl0z4w9m0>v zf7y?BZ{gm1)Yc_tHS|y-U9j$x6(rceT=e;Ah;Wz4#dbJ+qhS77rKRZE@97#qZke4iF zizU9LFyNfJ-5q_;{M{bp#aSSY44<6$U5C)t(rbg|OMCDhc~8bO@5}wy6`dT{bKE!K zk|sBS$Sm|FOZNqCwoS}B>bZ1Y!uzzGPxBqZzQX za_!(`a?@WV0eU*Nxm>6kEfMJ^N0^1>S84r(R*n*En)U zr`2>mtB-x@)$_wuaLTdeY?U`0hmTBB;pAlR9`Z$COetYBGtZE*6^qs?AAw_%^^rE= z-J|+F-B`K>Svd(R&fx8)JH(;@X7wWfC(s>0nAPiszT}SD@AYA{Rmf!+3#4Z?z97N^ zt?1f+AjiBxI^xIDSb4{HO%_@D4cL7GlCT!078VvBhy)st!Gyq{lJDPF%RUCT$F|kC z`1Pz4*NSarI*XRD>soD5W$Ib}LBomo&7kQay#Ho#jRVM|M$6PLrs024%F|JHs^~@7 ze{zG@aIf37Nm;5Du)8*3BVp{e=f83uhqXv=B-4W&fhuTUv$P=6Au}guZ|`++Sy==3 zeCN%hc++Fz*A?yeNs|jr+~=2PnvDdtBk!Nq!{6!nyj#{U6m_JTF5Oz!8kac#zv-87`+{T8FGO9JBaBq1xp^Wrc2k2_fQc3u=Vn{>(HNV!Q) zI?>%LXwltMOQX?kCAyPNwNaDB+{}h1@L=Bgpivvjk0Wwd1p{Sp@TJw&ylQf7iGdko zVOeUrx+Jk9A+D5MvXss8VJR$Ca8|}Wox7F@PX6~$Kc@AVeF&z0G+ zJfp&}eU@Dn zmr3O;!+^nB1>ttbHtz8 zgCx0{tRiR?5Q&KnYU9HhSlPH7XSCO$(*2E=AI>9&&R_FAtAwJxulky++0X~$fBJ9c zRT8&`F7XQ(rl6f7a_;@`#=9Kc#kI}5J?g(3b3)G#t(J2hrO{4P&U5NbS@nF|1~28` zt+c|!`K7ieH!5+RHaSG*^HMa8lb38HU?C90Hh(;AEe@7E{$q{fwGF0z?zd?rdPW!> zpJA&TBenUnu6&r`yo+bMRmEo%sz}p*>9FA+_7NU=`OdbVNO=tyf za^?cWB3{_v)2G8!57A?)$4sV~y=8!KqPVOqNt*25;MP0cvST3`U(XwuR%?^u8?XBM z5`}|xb3rZ?u*Lpa3?771t{OIG*4`MXIX7$}zHI0~Z2O6;`RrnQ&5RE0Lwp0?EMZeo)ciz_yD0A*3} zUaDnma-%ttEwa%krf%4BNxx-Q-oGMvb1hAdP zMF_)I2v~4p8(!T!IyF>P7ORG>9H{hlDJVIye4wEZsD%-b8wK2NMUStp4Zvv!2Yi{+ zIVZK?$LnHaAltBYJ>zSZz({o-X01I;sx4KDE=S6@Zs1bp15`USJjB7t+{}Bm)9$rq zd&A6yp|kOrr<^GsWgVA3)oP3oe;KgQ_wy8n^6>3vz(D@hnMhEvgDmz3i2yRM)l}91 zVR5oytKo?03y2_~r=PlsztQ^#?@?vm>zNlBGg6iniT8DkVFpLe-s}`h5IjG+ot&pD zcx3{yH6>s3jO|1LH*fW7{jsBHa`^@OTPRQ5PZuD6n#x)~A=!+(Zsh%dk3 zZ1cmu^4bvAfAF|>b44IGjFfE~U!DM|hAYGsd}_|Sum$M;^zyfPPPd&H((z02krW61 zlee1e#Yq9(W2{^ctW->MV!M{+J0P@%3mT$z)9VV;2SdMCcf3CpBhDK*QI%m)QZZfM z#Mm%%L)BbPCGB3@kfKL!25HEkmd*0;c+s{~Z>n36@fI-jlgS#dAsO2Cr;i6B+=}g; zh)+}IOQHy_xu%8+nK_)Vi5;#LLZIT5hMY>=3*75b(w4eF--bP8!zTjdWTI!??pv|k z17=el;%@94N87^O5=6>>ZhI~|f^=)o1ohkf{0aFg>kH7`+T)>i>e$FaT+XS+-N~ma z9Ml#+*gTNp(p_6(y0{$NgfPUG6&Iq-GqzAFyvTXwB(RnWV-8!*XHER1GGFEJ6hHKtxWtS zw00^+oOPVf&QMdL%D)Pg1vy!2bQO4UWGo+>W{XQ2bZq3ifu?M}lsl5x&@t#_X{fx3 zIz*h;l0A<0M~az*ZYK!tdJmLIBOvZH_-&7RGqa5Dp24ChAy?e2RL?dM&2w}sl!gxX zaPh{X;^gCkluhE^1SxMTIVEXe1-NHLjg%-UtGk*H&cIg%r+0L5VF6BZ3(zk!MUSQY z2=w-JZhbs^kd+1m#F%O~nj)e-lGZUCq3+&LiyYObw0mQ#Bw-_KJ8IS@vY8hw^E~QY zceEH;Ull(fUjKMs@h}qa#TNwWIFQ0z=j}n-8)gb3)sE>>JjanA2n5-~ilZBSYt4Q& zpT?K{@msHxL1NIFGhzv@{mo=-=%#dI0n)#Z-)L3XcXPy2rq6OUvQ0TVF3i%UrBTU> zTEBlSM|0HKcgH({T_*0O_G7O@rw*hG>^C_b!TTO4t1Ap{}`qjMM;v74ahd`CuWBfCb3qSYy z)wE5C(z`;1;R9SHTt<4m4rJL3kuMJcs^O-;HoCL($aooNl-=nmBrX`ge4vKrRjl`^mE;w24qdI1JjZF;{Gl|Bt#&zDr$0?Hp?8uEP zNZ7S?mZ6POFHdI3TBZK9;6&g|#h3^p*Va}A92j|lB#2^Ll6WZr*|Og(znDHmDd~mt zPLeYsgtI%eHv97w9dB0NSA>r4p`~y&DbdTaKG$WI%`P|*IFDIyRVsO+$% zNi>fyJNaLaskSsXLl^ORzrkjC>;7;HlP)i=8`kGcWPh!+%>pahO3cLGpX-K6o*%N^ z4E#ArK-UDU^*!&eP_BbNT&mPSmzzqqv5%-(+gbZr_gR4ero3-FXVAwkUF3#kW^ikq z9iJTNMs-nfnN+*pjo}Xc=%}dE>N8ZL)Xi-78qn$3>L-9ut}0zSI`Gw?(VM@ZqgO+h z(l{7bV-ZnF-812-K28vZvq|L842D@*mGYK@MowXkE*D zogTH#BDH`*pjb{vYa&Hv`wUDHUj;_O6$~!D1z6JvMlzy;<8*hl_+0KVsCo@hierg; z{H8LC-XZd2-^`~VJT|L+L~4-@iyeKUK%fA!%^R!@ggazsB&G5B3qJ!cd$RXgR{A)5 z#kP1T32fd35zhDithDC?d&q z8t1ovI%Y$yep#neNkfep?cH>LB5!@hPuRES z{OGhZ_u~uuU}DS#5>=fScKb^WvFA0)s3sv>FbT>-IQF~hnC-R-#IJ^nJat+6=#oHat^=vhr~a$I+S zrKN|S2X^cbBG*6QnWcxfc9#&n^jK~?`ue1X_Z>i9zzKgR=7d1~bl7=t(_V*NBsq_kWVuy(Yh?~oh=a8qp$N8e;klEMZ$B4D< zllSRtn;JWZfa!{>W{S^MI($~MS%xZ$ejt#T+D@+E6IH9$B!8JMAJuHrOG0}R2zm*A zm0lU>ZxS6T-6?oGbht^|a3yQjIkEd_?U93OwNl2Y`%Ge~{{WDvA#TchipbV1yl(py zhZs#m?|j#N<^}5mOkWMI}cdmZ*Ah8y0KY5=pDO{^RK5d?+ z`i6~ScLp@8`;p3ruTX4T85;+OvPmnSWv42sGy9YVU-}kTt69eZoUePjgH+qxkjXx| zUW1;ZKAU#t>_Y}+SZEhDA^Z|jH*6|q3@dR$*X@=KD!p~e&Whp;N^Af!@heUVf9W{R zY|-Oogxc!Iz!kz5RBzD6$<>)(IJC=ZYYW+*&R4YyV7>yBq->Z^J=anlwJ?(i$O$pI zJmLL_kzi`a6KcM8RA5{9Q1vGjo__x|zgNO!H*J07u$i9RGqiB_mEw(>SbaH;<#U|h zp9x4?DQ~u^X$N3#HZsZ1q45ZHMN9f>kAB7;ucsTosj=Zdmje^1h&ucs_ z)p}43nXGIlfF&lR!s$6%&zUfzc_P}68VE_Bmr8+q%*UgO^hsynAa&&D(7u~WM3C0rXxMy=S?JL8A+h&2_{Pt5( z*Ao8i$$Dcv%8Pv1QNr4%rr#RJ(cViJ8Q(6G)Jh%BXtuj<`47OQSA5x4# z1!QTXH#|-)$ouX+nMP13Whu8PRdKn@NcON6#YyXujMGIReQy-ZYkT13;()i1qU63Y zSo6aR`>AzuU-``U{`!#06Hm+i@3Usi7Jro*M~5EyFT21_jR!G5c!Z=Mfh@T}^$&}R zIeWYq)H_T<8WSL{$DriELk%<0_)9?c0QAkJO&_78+f@*vAqNtFFo_|`M#M|6@F5KY z$r!g=z||G)wmn*a`~0D?nI{{J3P$3qV|}f2JNPbk-dbYk4 ze;%sN-{2tFg>Fmq+Jh!SCI2PC{!3#Fu_u#cbsuX|Jg=+!;My&HO;BjKny);|Lv+bm=VLH^P__k8WE&i zM{h0d@c76-_wny^N6)v{ShdiLvL0j{nQ(+twbs+UpOGlK|9n#5tiI9Uf6!!0fwM}i z)UeW+X>CzgnG0iASudu;xO9G8+SV4h)Do+QI^!g?5;w5K0ZErctWqL~vMe>w{>1Y) z7hD2gD_c&L@;e9rsK@FDtH5a^#(Hu1uBei;0-be^NRKb zs{9+Q`8Usbf6hQOX47IZ+~2Xok+CixagIJQROh2UVe8$fsmG_LpRjz!k_z6iszVSgEmn+bFUdD7bQKYI?6+2xno0MYJIn^ki-l-3kO%Z%g? zC*TKgSi-rrNY0M`n>Fdry#}Ffns4I<$wdAYL9eAUGriHe-ocuQ&Dbnx2?Sz*bn!rH5dV1l8^ zym~XkeDe16pSEjnOO>FRYx^@6Ts?Rw2i$Ot+v<8j_#4DOZ_`KsjW52a=sXLGY_!M# zomAVY=x>1V_k;|2w&SDaCbBb%5voA6IOTzg1p)!*xZ2gP1@4TG547@(p0*^ODD|MblL>Ag8$5C%WEOLYK-^r#w>zV$%hUIHp;6R?7EA_CLe!S-HpZ_rhmuRg_0-z>)ZU)PSu;tw z?2%$wmqc@RAHHQ~&`^G?iPRIgGA96uZ|>5s!B(RqW&}o`rANwwzGZ41%Ekm^+uS1A zmVEy}&%v~$-=u*b1>{Kefs6P8gkCynT*l_+&LEMVKj+omIxc~Ji(P=qB2GHRfb?!= zl^ci&ud`Sj0vbyP(BnWQLV8Nd$G0G@d{yNuBo2q*S%g6gWE_cp0TTXuKkhPq@e~B_ z$3myMf|F31c?0RkQ>l|>yNzEwBN%5u%eol2FGngQvDaHbvmFYb{8@he4d^%~;;Vco zkDPU@f&{n>0}t#SmE%ir2+S1T}KN4VT4irQRs#js@CBZm&2q`vi*9r=B4H@#fiTs)@tFt)orV~3|O!5dSw<3|R8 zF54`yTTcR2Zjl!RhDN)+VblJ+)x!aKpmu30aqb*DR~FvPc+@(&4Ymt7=YJz~NK`OO zX|C!>LMAT9bstpR<4i%ke4a4fhpe}6?Q`YGYn`ckz@LUWj#(b^<1zMy<{dO-o8}e` zBYoVn+TIC@d{?V^4?NfyaP1C=B`Bu54*7j$K3kC0p4ZXZb&3ZsBV2o=l`A-Ekw)L> z?cJ#Qner?|S)Bi^>_PmET0`}BMG&qIUUcAIG{20!xE9JdTm^Yyk7%RO2XiwsJMKrS zTH|6?&)xp?2e&^T%oC_yG+v8{W%-vcPWHtCU^W)y2dACAiViBDg=7^wb>zJVJsY@; zSLJL!zz}&`>DD&)nQ(+@0B6C`OT>szfmWGco7GkvvEiWWHsHI-k@&D3Hvep1Yk17) z56JQt9N69BFroN?YgIw~Cn(;I?=-2+Y~flyXzOk0$IqL!<}WKbyj>$6RJ(=|Xtm4^ z$-^f>K)dD*K1ohc{(|idR+;O-ZBeHz59xhSbcdi@X!m5d2z)E(9S{6(65^IMjAB*dQ584Ts;V(oxQ)-p1Gx|2ARkUL z;|H?9fB1zGfRycAU%UXiVH(JymoA1M?HP&ZH4aEt26Gw^<>^BfEqJ)g)J>1HG;fep zM=RXA27Y6=yUcS+1pI(A&pU(oQ5<9=WU={!Nb5uT3^ciJK1YS5LW7)3d{B{ z1i9Qt^yI#DVPn0a}@y^v6%RPd;YzM!V%|L7AX5Ycf29@m->|Uga zGS=8U!54s!OYB;kY39peks`ODO3ENKyCMK6P%t_8W9LDp@|_1)CkZ%N%7~K$2uFRP znvQ<%WLT%qXSB#jVq~Q9*BJE{(nE8S}#7Kbrq@s#&Ygz8evF`y<9 zeyg80Y(`o$DYz1VwlLt6Sk&lB2zw0n5d1NDhu=gSw*xobyTGoJ`A2)+SBgP*?BJIO zU}KwbfDIVC8AbYXk_k5b#ucn6OLvft{C=KbV})F;DRe&xONJA(o6I~KVe{AdTHc=` zr3A1h7yD2KikJEd?XSgoN9Z33M$(4gg^3hR zLg!bzw@>w%#Csnh`w+%q#+z;Su0_2zNtPNk+8pTs^0!~HqkS3#n3wb@6>571D;~1P zy!3ul@DLx3aDaJ;Z+gH~?*RUeA=Wn%K z>MPk*u!TGdChgN;Nj#o0i|;4|3hL4;1d;P|k5xfH{##2UCyU`kj3z1bo3;1T;pH;I zwoadvc4IbW&m&O(?XyvQzGLId_SvqTGT_<3KD`t{N~{nZf_Fm`et*NGOg@ z6lvx0Up7r*#~Qw=M;|eRU$x9?x&?ER1R5PG{~|68EMYO!4~yChqi>B}EAQ2dz+fNx zKV+0USa&HvXf)G76OO?0V%Atat$AyfCT>Q3fikcXF36!Map;VG39t%k)P5oczAKxQ zSO(YfTZmWqn*UzD4X-;_WCtW*S@?t?8{jp|wPAhaeS7_$s#2zd4D6Fp6+V-@W?PBM zVX=q#;&>X{Bq>gGu(-=gWyDE-KId26zHlCHo#4_?P{^YouGJ4j^@sl=`gddVpCJRu z6jflafvnkqn`}BCC~LMp75_G9^Z&;x{MV0d+4NyBCtoPD9|HUSsvluq|28f3?->BW zrWyF8o{gPbKl@i8g+{iJdQdeqKnEOmC}3*c`ZVqX_KkmixImiczy2dQL=)0NtSNCx|wi)SlA_&{EndYa`J zH%Qs)_HYDd^{E+$K`_f=Al(FvT0#YJfOGZ-9A_(V{Ix+Q zA1|(HFeXG7&_R7M;0&etOPS`J8~*a5xK5Q0<-sm>sBczVr~8)R$~r)EA#{8J%u{KS z*{m1cu!Tk$S#55xXZU*0DF$b{Ikv{!imu=|Z-UqYmjl5)j%~{k`Gsne+j3GDtyHY< z`hbIJ$;!9=dvNaj)yc}h&ZTDgXtNZ~UO|OSn*vIj5EwsYVr?y~`#EVXJrd-GP@(+} z2Kz%DEO)`X+de~6itudygtGZdBxJb4NrE4-+#ZnPyj4&)HTK#2s*6ttK4 zw=$^Aez8>M(kD|@`&?B0D~JK{LB2F@=w1V^drU1l1dRhG@+OWmu&|s1*-DB>{h=u* zz6n6-)!W|Z#WVnd7wj6JEr`=_>(F#2D~}KQgsX)2ntdP}iQV*90Pf#AYB_-0O{a`T zf`W1p8s;(S)H$aOCfej|gB{?gy6b5E`8{Bt1iU7s%10&V_L8nd;AgC6{{BoKT$hrQ7=INsKZr!x3jO`}MgGcamYl(G7Uq)X)WO+V!45Y;2+*%Pf_NpHp*o^p z>DI@G+?toTB&@xQzr%7s{8^HWYCdVBQdnx=ch2hITEL2kn$o^-{NiECd`~5BDM%zi z;?h$DQ<@LKb0=s1`nN!l%m*Is`K77i4r*aY&_v()i+3h1>1(FvB995YM;gjS-hD~t zatnj1O#+~1x91p*F}CN2hZmzMl0tV)f-)IS$d@dTrU{um3yIpqBrwv2GRX%exomw# zpqqZon|Oh;GjDD1xnK{F$1SW6!26O~FcUP( zT%8r#{h@BtV9I*iXdh!V&vw751Dm~l-MU*f{r9L>{PiNl_VbXb8Tg> zC1gtSo9t~1=Ydow(>DoFo(U!=*nTB`{lrs7Hr^$ZL2^uFPd(dUEh6 z2;Z7gNjdVXmGJ^TBRW1M=sqzYR34|<2L;?9y)ZC%lWg@B%3hG9x9+492tC+#rxmVR zWm^~Wc3Wd6&$Fy!$MUq^rQfFH;psDO?Y?|5A zuGfBVDeg zzWkd;gpli(a+GPLaejb!W^x)@S3EOQvrZc>08OOW>C$Ip+eJw+6hra}Qjj@=TP+e( zcMJ@I)9+38VUv@`LGjja54M;(>XYEq@DnQO;mqz%5IMGGGqX>zh(CXC))tIbE`x%m zIaNIGQ@s0>t1cRfErO{IWi5l33{^i-6scdCI!TM?O!&;LUzQRnzx|rPYz2d+i}_mV z{d0w@6H>n?gTe5^L5&-XIfTB_A(;zb7p5?o2^qFOL688|_UBYX=>Tcj6U2JyB1}Kr zsM8_YKY=8}{%g5r=U7mZSJZDm{j6H*r4@fJV?I9Ijal0&{Iy4qFm;g6$Y8?XPo*G8 zuk9^qp}ox*x>cZH2!u33hgGeR!Ci~?gya-%F44n1LaG5i1r7;NXU+3Rp0YBRtaY*% zkXh7O#CC-?fq^&!f+D=Xp>cvBSKs1@`(v|)&O)niJ!$WtG~6)t^;-p|DWb9znXbUx z;dUn8C9klBlkP^z$ZQ9R|M)>;rqLpcknS!ltWB%Jju&?mcuG8HP4r&)dvrKnk!`9e z@XuexC9XV+ExuE=%39)Ui$)y;^AC!k+!7Vc)X~7-jU{#kbOD|9jz7}@-UiflMo#(- zoc21rH(|5KvT*m~%Je&1Azk~)&-C(0ZC~(Z9iPCokExro_q1waF4Ggiw~zJA#qiI6 zyDV+@M*|F2iy`8UuC#!{&!zNT8Yv}jweRi$xu?BDWwLW06tZx-lzYg|4!#xV#P`64 zs@LWAwR%AD4DDu{Ask!gJ*JOHXj=Obp0D;hnew64hwhGhJrTbi*sPEsZ4bER7{}h} z^L)}nw;-H%;R54;{I?N(hsLtXnBjk zlMfI{J2Ik5!RW${WhmBV>GO$AdvP=O18}Xltg|UmhX-4(89Cp`-W#XrcD_T6?Ru`E z|Fq}gNz;{6zY%TTt&ZC?dyOTvJOk*)LnHTpNsO1 z2z~^n=a++=96u$=RI(-|d_VsLp#*l!;Hcj#{$V9Z-E9RE|2+ve7wLb?o492c-&p8v zU@RO$#58RtO*6ot=-)s}1=p22eaec}jk&~%;qTYInOK|xl&?YC>5iN=CTdvE78A>l zNTgIRQC;eg*okMBbq0%~S*Mx6_+T#@2n;{xpW+*Mwzj-SK$`^Pk2O$ZH?L2W&BTTU z)zs(Es`~;tjh9+K?Wv4QY&gC4Xa{hl1r>NN+0k4Jyw_XJ+zPzwj6I-Ds|cC#oW1uE&6Jg|TvW zU>0j_c3x$3I1zf_uxzKC;SPW|2I*@}W+t8Zdvhzvey43K7WK14Xte|)L9&1aA0C&b zyBLOw3FRB4uh*AFoF7Z0@o^vaSLm#ZR> z_+fUMEo2Le^M>}&!JU*!U4eT@QUZY7&TrX~&fEgC4w^JFw!g0lSUe9@FvL1Ncy#u# zGc@b1Dm)c74uZ!*4Hor4nhy6R<21_Xi`iu!*vsAj0E@SxGn02Ms4N-EJ2c8Uw1PqB z4I-WegpFAbhbHft*hdC41#Lfi1GVZ7z$~6^e;uGX`s%PpU-p3iSbby@K7ltV%K_3R zwgN{E!K|1C;#N+(U>pvu_iPC60X}{JD{&d`Ba8>a->Cy;LTJhdjC~Gy2?Z^Le#?L? z9}eJr!PJ29DncblF(m7wra@8&9ss?1iX*aAL>d|s5pdzW0SR7x$#Vm&m{7M)`8W847e;#BfG)ZnkwHzF*rD(S zJOAH>QvF4Uf>2JVX+6QML3zu}jucf8oI)CxW<+5M>huxq$)_Dlqmb2(BGwgT5fmrX zMy>YkYD-06CI!Y1^tmn`o(cr%4yCaJ=eQ6Qq+3cyJ7YKI zF~Q#&o*^~Bt^NoayZ1lX`?iUBLko?11J=WZFiBsaW26eBO(p-qcgOKaUpETi(&_6tdyvyq8USc%9-oY$?8Jb}9 zKVBO;SJV}p9n_N>uR?vS3cM$QGdOo9u-mBsEDeL7n5#zIh}8c!lMH(N(9D^?V5fa* z6!Q|NSLQ$r1U1Zp@9lw^5Xxx-{6d_Bae6-%j5qxNrSCjHD)-TS_;P~wN=$ti-wcw2p0p^x;G>mk45;ods%h`q`)L$cS3O^{IWEIF`%fWCy;P%9(M|idr%o6 z391pw{UA^5p{?VQ|Lu-VWx_JWRlf`3zx!HiC>$5x$72Y9e2A>fTdBv4Wd#5 zz%SL=Ys)jDx~IxJwlX#I0lyYrKj{lKKV^U^YXIr6M%LDDJlc8#c>_p$5%amySd|$A zhlL0m#bHNx1wS<_XDBnqjcQtF0JqN;Oo--F{qh#deu66DJsAnNgM91>6}S5fOPwu4 zwo>7EBNHhSvt!)LQ|)8DMx$IXap05Lc4wrhMW%Qd!Z*XM4}jeQpZA1zZc%hXpmX6E z%rKCDYJGF)+|b*%Z)0YxRN9^=6?R$8r$+x8n!xK+X*>2Y&Opv!&{L&uYvR_f%tvyY z%CEWGq%e>I_OJkQ(kL{sp72?5)SwoulN8ObJ2VEN?&1~HyTj7fR`_3-ls7-T&&mOS z0cNT^NVC2k%w-JuKa9O+R8w2qHX3k86hw3jA}FAbA~m20NCyidy@Q6P8!4em3m{b# zQ4w%crPoj;K_K)ZvIQa1OXvYHgceFDG1PA^_xqjqIb)n3hyBALj!o8@tTpF-U-wl& zhC?_gW8l(BV6$;XTL+K?Kvl6-Vv1&TTV;+0D_NC+ol;};5v2!vn9q)O8Uve2fe<)@ zL%$Rxej99zJ>0wBgOSF4c#(w-ulnsn5*Gr%sH$70s;l7_h5yamum#|K62hdY_C%Cb zX>S?cnRwLr_{s+Gk6(;xU1EZUD!Aw_9Q9 zXC&pOjw)&c$FL;GcT8C2*>w_pD($!9E$~itY%i<;6N#gYgc5nLxseug;5gep_!)l6 z0WAJt?p3Klq5~&!f(i~RS62frmE4!4rU8eLpeQhabfZM<3igiM8U_J(tse+b%mC?m z$0JJTR())X9yn}XX$Oq_{vF61(MNpRWz?@koL+`-yxu|+D+#VRV6^gwF`@)=K<4}x zec1xr%tW+Rz=4xiDVMq77|6X{PS_iW`uTFfL{VP}NM6u;X z-lf`e23igF<24q}I>>yVA z`1%Eo-=nMd32?Tvd9J`mw-;Ue2y)VygN}JNbT^=(vxlWS&_%9;S1ccV>Z|L@$IuY6 z@lA0^XVY2qZ5aAnlvCs)Po{vhDc#(uLl;rcsKgR{hh$CHZlpdEgH;{{Te4JsSbj<& z9~7a;0_#o0u%f3JPci=9WJTE#MxT&P{t5KKe5Vm4dfoVQ8AJ@_LIup)Z_~NO0m3hA ze$Fev+dco@tE=fhiah|d{CPuVXmM~u)ux848p2b6N}<*g=7@maY8Mdjhz=aJO9GgB z7S({N$GvP~lMcBgJt>1M?)0HOnKJ8}mO+2y5FfHz9dZNkyyh|w8JRES)wVa=Kyfwj zLf=u>iB~sBzX&D63HB)a(cLT=4@qXV8KZ&f;x~nJ503vszUNPILf75e;LV-$wS&|_ zCtx0T0TkS3QZ{H|U6j)7LqHu0z>wx9WgD?$EpSDf(upLg5yoBWbXYp*&kA@fDtTBv zf4jUw4{BPKKGS))8|U3lQGoi%@4_Ai0ykK^Qf;6MgHmS)DgoN~NbjH#NTj#VdC;I3&vkke zU0D$g`Qth#iCa{Oe;ANUPwl?WUFsvr3`4eo9m=5{m?T!%KG9>+k+Dk^+gAZGvSPpf zfUN^R8PEkqJ#Fb0Fly@I`)KIksuXNzuC26T^3TD#wW?`xT(AIv~4@N8a)l**en*WP#x=1NMF_i|~D z3${rBu>B1P2`9W)_VK`wmY%9+0W7Q}sbFW)%zAgJ@OzfMr?F2Q9OeP+XTX9cFj~49 z`i(;uLEE)2aekSTdpYfC%Q|@5@9g^ETe_=8;$461mj4soqp&0nbOdXfYZX3) zSMP!RZ)z&nS9K0`!9Am}`6kDO2+Twz%-|d+jGb;Q50x({kA8**xfK`}xCYDbAO@cS zZ2I`zl^VvusguG>zKRXeM`5oO!QKEwK=)g^JJSUUPJp&gkpx!^Kr2eym2KpVFlee`M6xrjyIU2FuDSYL<~Yk7l%q- z2(qITOLKi%@6>dZ1saT5#$k8|bspAj$_Fd>xGkh84%K06<=MeZrl*cWohWH?73F+f zDx{|ulL#F^Qerf z@}U%X6wlVHJOJa&CyLt`XD4E_%2-IT^KAdepm32Wd*z3th9?yiF6Mtvi~P)X^}mp!0xx*AE~F&xw^0F>l3iU zZ1W!MQB*nRE=v5cz6$vq>XM+ySotk=Of#@g?M}VvZmp}Ssp-jgQzuon6zZ@3CP=I5 zv=5hq5so6YYT!X;ZdJ!!fF?}p|C)$*OX5d(0l9eP?t}_>pBGChM=^#=zW}zKU`b4W z(XE>)1DuXe4C_2ydPTo1fs@hG*d0`eVxO^%z;xa?Ov`!oTOu(GK+ktpb2K&o`C%J? z#+I4!QCmT#$xWLDch{iEEyR zULP#Zs*gK8{1t4dAg!uYvp0Z_#ug~PY8If=)hIe1>fhV*jcA)U~dH; z3}yw@UHqFl62yJH?SZ>;ihFVTc|6Z=V@5CJDcpny$MdUxo@Hs@W4HI%Rst_=x*TM!$~-aYVoQXl6gw^^5-;HbCw{#)`tCQsqQ*X2MbOi>l)+ zFLV26@A7Hv`i$2o5fAoPoqna%1++DvUFg0v5OCVjZ6)IE zhAdKDQ$O)RF2_Fj+5c;#A#1w}&e}11mmbL8Rp8k>*S2k_V&)i(4*8#g89ww5(aZZU zOXg606xjDv>|d@uI3}c*=1A!PTEk_jY7$`akIN2N0>fY2Lig{@%DstUuL#l0KLF|L z2#Wa8Q3aCM=4-Tj4DPu@70)I#a2zS9bXJrR6CE?#SFH8&>t(`<=D8);8d*;t3@#fD!+=U&^_5*Db3Kr@m&HYwr zL23{_huaKFh`?P`4a9QV1avWgbXC;WMM!P+-r*Qshm^nm*!_~sa>&)I=P?qB{@z9w zidf17l>~SJ>J5AML#ak?2ps2xN}adS{1+E49p68MeT(0rf^!N?d_+h5M=tFOy+~gy ziF+i}y9ZHPe|!1Px(NHtvYfU98$5bB?5+y;BQTpS;M0b7UIAzTbfF45O+`cYF2;A> zaJx@bE$HXxx`L+E+jsE-ZoV;ma4)F7WqsqKfHxm!rIX7s8=Q{tF=5t3b#e>jLML}& z(0fVLCYSb^QpE?%i1~JNRxmELuv|8|oxSO5eSb1do5?vTnBK1@yBBC2IzWYz_*y0u zKqg9sEs)$-alh)=Gz&16&1@Mu{ARYiN#%dFk)=PdkX8{np4oDg37wwue`@^QXi zy3-0s#{o87gKt>T)<%jok0>PxyGBP9fwPHa8Pv!Bq4-?35I{ODCia$(zQIQ?aG&$U zchfXmK0<~%VBbjwR4&1Yr!dJgkfm){(I4E$hMM+*Zj#THcELG*4+9?;g%z`|MSpO$Tim1FqL5t~yBv&e+8;KyLnLpwIe4d{hGn7$;5e@p$u#@^#1tLj@!aVKxmiqn^23Y=l+IqWzckf0FYv|L8AC*9RRVZY%fJMN;HwxxWc(0z*4kt zcZ!w;nRnSu98SX{GE95ZFF3|L;&oQlq2Ecu4#x(lMB z=N4g=65PxGgPeClO`rbn(hwj-!(!TTpdVEcvVIEq0QoL|+oUV_lplFtYNFdy#Yyzd zc4`wD$;(P+R(EnUwTlk407Vu8C%l$d{w~GRRcoUR5DUs?0nupW6G5L!px$))7B6Nj zGko_`m@(@I8QGdS(~23V8Av7_-I}xW(PYgazJ1YMX22^(^u@m_(g6(Xk9$K|APXYI zrM<3gQE~ut2xOO48#gdsM(+(7xdBMs{<%B>tt_+Vc9K{<~EMfwG6aUHtgj@?4<0ZJR8BuE?rd7AGfp#W7xOgma!CIQl zi%HSELYfOcv>Ar9ebJwU@YPcwJ{1$_yb1qk2aV;#1Gbls=Pzm}2tiYBTkcc7@$JQY z6DF)HZ~9j$&HcDU{hXAmdOjzkwwQQNj^wi`G=nr=Fx`;&kS5yv1=1HN9(3pj|J5Gd zZAwZK?zEuqr-JZOcC#ggrDyl}Zi4d!JR53{fh}j2`aDxOze^GNWy*&rUO=Mf|7fM* zPNFXnC2ONo3Wx4wti8CYe6G(^`Qw?WNm<^#h>Kbga|{lq$!F30X*K>@ULUp#ynj)0 zrSqH=%qiJEmIDpQ9+~QrpqD)&PX2Y(B#T%@Zj7V^>b8e*S;hX}LMgb+>)qc&7E6z| z0C#WS(f@9x$?T=rP;t|iKJ%xcF+a(?I4!mpg^W#FB)EI%J;w8gX-2K@XX1@7z4 z28F67+q&jTKqlNB@GrJ^UW;=5@N2;?0X@tq%5ekl+OcabD9`j>asS*Q0A zy^Z1$h15s_DEdM19zF1h zfKnAa><93umN6M9Y=S+p-c{Pf%HfT1gg`^eY=IA8Qn#e| zc#9(jo*j7X-9niu7$b;Pok_!U1vtp!nyvEP|5Lag?AH1^&+YeYr*u@kvyN8Oo&wuCshIPXpd1)juFO=a=HR>8ke;7QQ;Sau_U~d>$b33g zv%%g&^mlHG9xyT@C+8^$O{yB&i6>b7!D;qY2g{@VVZJ|{|-RxYbHu8iGHsGpP< zW{)1%(4LA^6RqR^fBTcn;+Reuhi|^1^8XPo@D8|ZYbd|&y>kmOaO8xlUr_}2%B2GS z_b+Q%L}yqORI2p`?+AD4<8?M4wUy?wsHiCa<3U+eeR|p#3$Cwr!;>Rgx`t|u;1H;g zso4}h^id9-`w^Dj<32IqSmyPa&R;6O|0Mt%IbEOkkA6*NtGWxnvpG%;lZyt5(0aZt z#e5Q}Em3|aZRKWPg2Nuk-TdV70ue60o`mCHD!=PH7jLlkc#YeAN%tBIPJh}Y$10a! zC()&e(oAs3Y4D{Co=Yk?s4MCw0?>Lx$ttCMCVG{0XQ7`H-;{4W`9dNj2U~41m#O`$ zV!mS6?IlLLzn|?BmbS`-h|$^Qd9xhxE!{y~UAzW~0_hU3eg>ta6PSkbh-*rRm)L)Y zR5QC%Z+{GnNJ>`k!x@&(auVO9>>C-|0CZVP7Bgq;W#!~q0>WfQzbDoJ#b~K^T%JY7 zN<(NB7Q{Tv0dvLCIj4}rG3yQMA&0qK{iYL%!-hAifamkKO{KXd{bR30M$ zb6tM0&#i9wNLuu%!jo^O6y03p4TL$DEh~yTmCAp(jyq811sYwjC48j+3!EJ*o(4cX ztEaWf=DG3TaF!c?%|YrMlzAZbicdQ;9i9ip;HXKM$o6V$J(Jf8I#U%7%v46#mFW|< zFVYVs0}h$5m?lsHNU0v(a@Mu?Pq^P$Y=t%5koG1VD}3<-^;*PW{nu^j*IADTvN>+D z=QieKQAb!=tv_5dk8Jwh z;KQ-QCqvluhsmbUe`(!LG8l z=)*BG=sB<`$K-qrRqjy$>r8xAIPfeevO}?NKnWAe3(!u}y`tMjTpXA1Z#fYhVwa(& zg;4D_>7*ZBAc$~wFYfgQkd3xet+Xn)Y42Qrzh~B;K?g%OlOItg><(jPM*KT9g{0Zg z*c5uYPT5BN5j`;Ydb7V4p<2260?;Aby+XfC1n*K4Rd<&w602F13o%o|kWEmF9)87y z7cfNevd}TA%t52f$m#i6~`=v9%CqTy81LWtAmBvXM#(6fVN zALntTx0f*kXy+VTdcb%^{mfOJ*OZ(KCjF!&mbVHy^<984d3S4TYrUk9P%XW&bStuc zZc`p)j5zKU+da!FGZKgui|Ap>RF6NuV$nD1jp>mNX(q?ekEy*rn+4O-;yO7+j&G|n zF)(P+AUknod-#2scDUc7GFkRh>+xEJa?u2eygryQ8^gq>-qLMD@-z3gJ|(8>PtmB3RQ-eD{6;$mXmT^n`D?lG)6pSb6Q;BJR_Qhz z(gi(^zaFAQOg%X+k=Nv>E1>hxTNhOB@$tU$fChNt{J7EcTO6lEuK_2y>n0;J9-x7<+4(FB|wu*znH*VmR2;dt+;73AV@xba8 zCg}6?+i8ML;QH5wBPrZg-hY#MZ)hX#vw>;(2ft#U!A0rw%0a3i03tOQII1F86mqYP zFn%pIo?T8m=*8Utx4ohiFEdiRxUy;=mTr3a*_q!n3h?85R{$O>Tm&v_+!KBSpues` z>jC`_(iRWQ5V!6m*#!Gv?Jack3{t}Nwnd$+7qjdtsR~F0pR_ls&6E71@LdH zEp1brEW5A3U1U`K_0l^4;6~jacz`7jYjKBz5(|5v&4!i;S;)0-=}w`th>wWu?`F8P zbn}>JB!G8dQB!E7?)UA2D}wwhg+<^ZGTQ}?7Yy)aCPtygw>m*=r3T>QyPo|X2!^39 zL2y0bI=~BcU(EShtW6-ma;gdt4Nyi(T%%~DIRgY|{HU#k2!OP@x;mZDXrSSi)#pr0 z2g)s7z`3{y5IXaGOV(q>-&301Nh%fvH;~nC>Eyb53~TfF2yktw(PUOMR#jH@1lo*! zb@IvxWAKw@OZS`Hom2m=-jl+Eyb7VDs&^**$6>dB|MrnK#WH!!8K96tgPP&0HpB${ zq;?Ys%3uJjb85ccO99k61T*!J`6qYpD91=2KP<@dL{~z}D)nx!V zC=I1Ce^SV2PVNUZ7M;%nARX*tacL=j|80`CG_I~rp>CsFk&0geN0m|nwv;eF0Nl|L zL>X3`SZ5@QV)e38ym{Hi>Z?tcv&#)WTB>2`jyHFj<%x$q&^mXo!1@CWlr9lS_O74wrxy#mns%HZK9&a$H zxCcq~u=c8ZYpquaOwv0(lYO=e6=^Nq#(E0b*T0Rz!_rZO_nEaKnt{9a+lW7m5JK*lKyd^Re?NS96_?|dfKZS>p^ z=pD)DC2;ZGvGsHMt_M2xHu9d{p)q0vEKBCY>MAQ9E={$CH9Fiom^bq9jZj9}B@yYf zA@V7zAx&Gnt;!;zeE+>z+AtF?VW^eg1VOBKaZNkLrbs#X9blt>-YR5&%Mf5$v$fy2 znR>Bz5NaWemYMn0qx5ee8IKeC81J@bNsjFakhjEj2pb0OAr5q3pa^gfW~xkil_%%7 zmWA=e2Qp}mATIYr>hrf-V5dmL0sD5({pP6ty8J~8tAksHFj$=wa4}BKkrIH>+7FWx zRsqDDNT2U(^^A|v+SxUt?|?;#f*jZ~MS?OwWM8lNgC88>Mfh=GJIm8ks`Csm#tIF) zy1yRDl>$Zyjv&cbHk|;pyE%YHKZ+k{MS-X*?}k749s_P*9~j%@f)+Mj+F&*FQX_qa zau*>1rne(5!5V!epnPqlUg~rmulnlsymO6VPP!WxvRCqpYm5+A9he%S!33NCa&V;^ zI-5`4@c40e>Cb&=k~S-^_L#GwemacpWB3So<4a3|$ct}np66ljQC2l?eGGi&>9^UZ zaf7I5TbXO?-yo`v7<;*F{rPdJ=_fm>*t z128&Vmg^c}$Vu<{UypU_XxTmP*O}95g+qhqdoP_&Qq{br&(~}jmgx!JJBuc_Nd*D~ zJzhNjdy)cEh`Ov1{f}|GznXLuvK9g6-N+#kl)`!Vsm5JH3>d;cfQe`>p=yyOs6(Va z4(}>V+Lx#5BdmfZEt$G{`FtlHGddH*5c(TytsE9N2Rqh{;%JBsZ~(d1w*6;Frq?g_ zS?PYnZbd#DZ$)feELLE5mG5KFod=`7> zYE>>bMK$*XF9V_a*b)@VYAwbHP9NS161A(>1J)X!_8K<2E0lf`ulZx^5})|bRmj|c`G7neUjZ?tPq`@qA zY;5d%U_v|5*b@kc*g5mke$m~p#1^~==+BL@BPubw;&ZWgg7i7D3ZnwL2 z|5Jx)jCC60-LI{ox*S{Q41spi5kb`uDye$lS<1Gp7}4QU-o~+*)2ym@E_DqA52(Z) zSo`sO$)jDx(E-y@ijwR4#d(I3^mA8F?1sVdy&KW}ejEEQ*psH3y82i2n_g3ZdJy|s zQ6fucJ@?qZ+X%2RU=+tUOTw*`-sf1}N_|q+?*B+b^VV68JYjl}@U=7jf$)+qFg?HT zPn))9cZsy8hdG|8CZlsT11d_epG$99Ccgxs1o#__w2l_Vwusr>L;y=Z!Dj2G9)oPV zZgf)Q06+Y=nSVpnm!sDtY)g_{WAP z4+E2<7rd1NgQx_Yz8`x9e8&?G+vgRPUnbU6klX;_lG^C ztlk>=xwXxhap+XN+~M`Kw)2Dub5L-C0WT;7V4=Bez>}c>gB>1Zx2lpLj@^-uXM)v* zK`fXtkl7~&5MIwN^#e% z_LbM1v*-`U^L(^R?1E3YQtaqUGsIz(fI=cx=b%s2A}Z|ZFwoGMN7m#enlp?xRcX# zvFNWW=3gHOZg|C4nJ8s96!r%!^kd9whZB3SqVv)G%45YnuS9uFPZw#8Ke4;=wjk}G z=k5~lrN6#I2|1uRf?k`?pGg zVqYBq-W8#iDc9C89_P3&l$Mru1?vA&;S?V5h?B-Ygq)M*~a!<2@gkF&%53Fi|qI@1wS@@&c#VWtoVn4M7m}E&b*muXAkvIXnyhY0PfYu5>(JZ!=70serA(HnU0o|ZdfuC7bUgm9-s?Yj4 zeb*T>Ptv*8LKzc&s`e}=>Z2r8(N(;#TW3~wp<$q{_D?H^L#3+&07H0-ZbXV|WC!jQ zK4I(yvNHC5$(CZI90L-LW3rdV6G1nNPB@@rR(%s}NDuYs(kV%3_c(NB_Uctv90PO! z-4i#2+4Xr2rCbFrk&>FY164aXr*5FEFw#hKLM$Z~9X%5i)7?ST3|2*D7!~$P6#$Gz z$~EpMsdh%H(X10Kkc|Fl+3p*xyjL-k$<^47Sf^X#8)h{$`tdaAfCAlX zXQ80oxYXR(*p}qKBVj@?YIZOXtsF`%TrxE@FK^0xqUDVEFdX+8q*%}eWr0I2|07zKo64<;+XPFU z;AJpU!i`?voD%cP-~D^|4<2`*93TjQcnKZf&x^N5Mbrwe-%N;r#PNioq0+ji?5pxh zk4i#|CB@XA%JJ-(JvAoYnq0SET2%EH_(ATM!%}svIK<8!0>0H)G#aTVm^HZ}zv|R9^lC|yhO61=2|o$Y zgc_INU|}Bk@Y3wP#M^#)9r5>8HLA$9*jr7jjiO^v^D$8DqOBOsS#hCuo7`aMjzXR)3|%HvUUAa z{s+dPI=e%M8?3LbfYZy_yW;UkOHtwca*>++iy7NRmSm#8;|a9yfbPwQO;2ZTTiDnc zgx*KYv0}`8SZVUg)A2FCSJXxMex9>R$id!s8H31MhJ=Yg=332zsl|qW+BJP3vyy-d zo!8{%}Wo~r`zP@#F@&$*AqvmQ!X9l%8>|oA2NPFl9Wc;EgblNwZE$jPy zPetS5UEk%Azj;``#kGyBdql;~9y(fHMsN_#`$ts9lC&l0d0T12#HF7KW6jZLI%QwI zk1@_8r>p=Q4|~;=lkKNzL3x6RGXm$uCSqc(g>dRcMdIKfbT*29!n$xFb`{gf7pfdaS!i3Y)DV3Rx!80;BwlXgfrGN7%g@|+?9T*0tRaXs>!@T*k3)$yutB&X@Onw z#2V}Agv(&b11M073g-!MnSZ_T!A}U-o}B4XB(KjZ6$D9{ZZ>oEjKo@DOTPM+0g= zpYC_EQysG(v~cjHS!0tguB|zsKXJEwt^Qj4gl|<4c}ydA^#;Nq@#oU3c5OwoD$h5P z1T&R{Tuiy2e0gco(k}%!Z9@6_z9_k_sWv(@9xX8INfi>;j+G>UcR-0KU+ImgbVu}{ zqnm=TKL4!x#@elW-FHh}+Q{DWLa@cm(1T}=Jk#6lm707Jz88$}Uu7}*Gw9XtGJ9Nr zN%G|{kVz&q;lX5A+!(?c-cs;hR6yxh+}UedC6()H77rNNtQ=hgvrlI$nh&; zD&Rk1l15X?zp= z6q}$w7K62;J+rSrs<%t@w4%H0DI0pGB$;I=P&@W5$->??2Ru4rUSl46$i=ZMxYR>e z;qc0+Ls!T1A*Jzq-(L${W~k)I^-ojfP{5W+Dt_`xKYv zBi;fDRBKS5?YKDXF6UB@g;uFDTOV8ZhZ*RdJHi>-CubIWK|tH~92lN>t(W8jfBE+c z+tD%2-kbA=1dE(4JnmMG^_DGqSMF2CZaen2tw$$hedZsq9&kBIq=fXJ z89xl~k}2TwND$Ns-U1l92v{5F41vJba~x9naB1al@%Iw$0Y8hXw`}?r)oxKqXYVEO z0#2oSIc|6!DhqukAR{gYpo=;CYg-3xnG(3J>3B`~QyylxtKrX;!@Gfl+9`cLj>JXf zW4PRc;ij7mEjG^L+uAL+BwImz+@F#nme=WAG&_jDi1tlV5E$ZRtXH=H(Zxrq;UrmCccJuF^(Q1kS$Qh_wh)#roQMayHR z|6vR)j~Z!n={kKaT>6;hQiIKiP68foNC`lgiY(|C$toGn29&KR3THcuK;R?)bMH)_ z27(AOvurZroka%1aJTir_0MuJSm6oCfB|4i&Mh@q9SUw~nuEQHMfp*F zhAb~H@A62-W-n=FvOaa|Ae#tT({1d_gKG}OLR20zpGz3EH0DfJzoOW`0OFl2VO4=o z`F`)YlDo1m_|&VDt!Pe;pCW3>S4p9x!14HoeuaZ}T{OZ_><-)x!1zurf59g=y&L8t zxX=JOsu=w6M) z0(5EMtM@u|b*xW1yKa)wP(}HlW02>I!!7#XU%Df1^XV`3xE1$C8HZSwInMhuH_b-O zMA_q3@ep9Jh2W+d+bW~uQ3Lr8v1fx6gj6*W7?7wvG&3v{p&X7~&*Q8lB+&T0oB+7m z2e^o5AtSp)Q0#rNXR?Lono1<-6D0%P?=rdhSY4n^lc4OWB{A}V&&O>{`L3KU5KdXf z(SG@5O_|~xn@8KI-%X=$(`A6hk^l^`Q2yGSHKs6gTN~5`q-s-ifmy?ppy&L0yFgE+ z)(}N)vV398S)jvNoL`4t&g57O)JKW`-g0Oa;Ie;q+`9}<=Ik}B{s?jMSIoG|Hq9ck z36jl96$ilJ+V;6oRNwq$`Bnc%de5)mWrq$Lm(2MrT7>Yl0hLNgKw%JQy)HN&S4S!a zo@=OdV{nrMy0`B6aC}tr9`V6;=18Gx!+i9}MvTg)=9^kT&iWSe zxOzVLp5@TnT0tKJWJzt>F&jPMLR9V5TGQI)TF&n3a&=R)~}TNJ0%xy-Eq<__Mh?`6e-JNpqQ>Es1ssKS3O;GKj$zvSleU3 zE@kn^#5Uuqg-zmHBvQ1u@M0PkCzNmpFw6%CRX2=K5<$;62Ni1`lChP0sw6V_n?6S6pX$;Gt(iZf}sk|L?8#&ghhvm6+FeY|54drJQGZzc19Gnd8pIK!C z!o8UQd}Y+ZctE= zl22O98=fk?ySWgliUxInwCGAeZzVM9{LdwY_CrQccHQ-?!{^de?iI%@6kQjUIf;o; zelt6`XiU6^kZ8Sbq!`v_Yx+>3P7s&>Nd|*L|*s>3wpTMuHR9MaV4U}l^Cis|TdR`j6Q`C&M30oJ zKA*S;JcPD@_HzsX$x1B3Y5WT7i8ict!kIT5qT#W(HKH$|VtwzOzv))#oFxrB5gU+S zL`LR_5`puqe71n2JbdJS(UGl1#GRHeiYqmjGX@IJg$PmCtMkjn!Z!~?n~cpqJ~-V2 z%?^5pKwk*{1oMc7^9tUMmCv@E*nXe_UC4{CEWx}6O3_?5;dCW6DI@FAu6v^O;AFdY z=DyMDnfV7!e~Unds${(uWq48KkQEh(d=S8Hue_n6*QcLSQ{lY?)eTwKAZ!)}GiiXZ z|KH~@WQ;XT?{s!{Mw}Wm1&J!-G!1_Le`qe8SDpbcR;i`jS=46wx2a=^b!x%SVh7T$9tk=W6(GV=slYYR5$oWJp$X zCbQBo>mGN48gz7p+E4iKimmr+OLl!M@sQ${w4QJY)TJ5|@Hzv($0_$`z4NJ(QPJ7! zI<40gOfv#?pfNazcW$9wp;}nMOSrO_+iHJW*uzcmma7r5ZI_0e6Kp6mM4er>Bs;~B zzThu(t)(TOu>gL48)648bsOjZd!~|9lBPLkP3UhUZkC7ZqzQhFTzsqXV!w=Kb7 z9Rm#-%~)~IL(Bp*wZ@WGg??HIa!Xm)ZR5&+?dIn#B&x(HG-VoNr8F;5-bb}{5aqI- z$$e3Eht{j^oKB~hRm<%k$x@>?n zS`gA8djsRJk;)CMCJ89{9RYW-ujK`c>mh$lEO55bm~D@#9xj5Q+x>-KV>VLP3T&M+ z!s2s+-gM4d9nR%^;={YUW}ZDpTfenhGkcN%!HM9h{$zQ}9KJZnY)T$Jmn#~9O=h+n z%ba6KQMKDO`<18~t1)ffLp!Wai}*RA7w$9bKIwpH2+{?J;@?U!OZU^h74 zzK>KR^S8VMgNEZm&e!onW<8+1?}eDDK$ulUv zdBIcuaBl-a!aFDZ`!v2k3bWG;$=G|ttQP!BwSop1MLkokr2Q4&^$YQlT8v!Mp@-5)(JU5ZN!Z`kw9+mYa1C>BCez)k|HZTI6Q2> zjiUx0b|)1e00T*@Vm81(Zb=g8>bx$Lo}sx0Hgy~0@Vc?x*}%ChIcQhqPAQ_o$S(?l zBa(5Pi55ij4bWi;SZv7f(VR~u+Qhc&)0R&2+f;IpY-9m-^UG}bs=ud&Dz5Cp%npiZ zvdJ?{`zh~`D%m8?U~Da6=?d}AHo@$z&UrnxohAF>hsQ1qMH{24^ev+ z$$R5)1tV;a_}EGpokQ00*{-a1y(;5B8IeXiyP96&OkXHWYVz4|VVH@qPh+Ax(U*K5 z@y8&Q5B9pQ3A1Za8D3XEQRSM7KfAcN$j5bvuKtwn`U#DbeNlw4rqbgBpg4w{rRVx>OLDr$8reZFG|OVXAu* z${!io?~?iVumM3cXvIf4mQ05hQ#`d=jt>yp$cP1CDY}oiQvZF6I2Muu>V%t&3*Vq< zr`pbGM4W}!$=WnIwq))eCg9;D<0FIw%K8>*xc1;zN95;!UR=Jk%IXd3vqYf<#RnYgen!4!qlLR& zm7f!t@6*7b5If&sEQycU;Ln(ZHGn?{eTiC zu^?gcNaqx_q;Sc2{WGfnXGa`D|7hvgm;_Gin6-+TTN6-4oM8`0?TJfBxdg-AdVudz zw;*~S_bRWs)oCHS-FKC3?jgGtgsl)1P1%UnPd8}sT-rl77$|MkTFH!8DT~{ zP5$%Q5kCYOn$he|u5FV$LC56z|3KtKGP2$`1&}+0GAk!6 z#zo3LoV8@5x^do}TsSL8x-8fn4a{3o$l14599ZuJb$E_uLh+k~Y9V%t~bPQ*lGFqe(b6BaK6(?4wuFOIU?Df3cz}45oF9 zsQ_dLj8wAZA@AH0-$G6!v^g%wWwnfp+c_tEh-_-bR8`4VWImxyBd@=(-&rwDDq*E9 z0QuUcVX7(s`3YWN>;0N9rTuBu9ZFiQO&YInPQPyZiaa{6Y^Y}Z+$W)<8ZvB_B5 zlG(PKa;l7nJfLe;;!vwG1=rp$ua;DQc6qas1k^Y0A-cKl_Be0H*}+oz2+OG;x)mYl ze;?Sf1W_$%S@UvoPMwXhomNwVD3iP-skdeL7qnv8Xc^+!QIV( zZmnXXe$0hEh%P#U@i<|@D|SP``mCgdHkcKwf|z5!Vms05n35V`70 z4S;MFipISCGpcK7_-!lpdwV`FIAO>M-^Gq87qY?BoD12EeLe|9yNa-uO#lZ!+4Svc zgQhy)RX2v>YJ+X9v6|5>>}~%w9XoBd&3UCS5igX~iXzg!Yz-~FX!}F*Wz1HQwY*Yc z>c43BGT?GZvoSmS!IgC|FxN{=@?h3Xe2CrXKk}r5U$*iP5Pcu=_m8CKB!1O;to?`s zh`x)1oE~0Z4QecziBUZm@hmX=yY6HiKdRJYt4T0GiAUDu0~umsGzfw&tbC%r*av|> z7@{B=UWpQbueXcB-ifNY3!dxEJwT!L5PD*>mH-{>v*^cipVVU3qX_MiUM&gZ@R!yB z2_#w1y7Q5DhuO-$UP(tP@fV6pqf zONG?Iw;k!KT)vD{6l3{ft z`vNZV}C&Y))tSE`S-DVnX3lbpqyK=*-GvUXlYFX zzY#U=L0Fdf0+>D=+=WcB+EW1ENr_SeNh%=)<5#a}XyS(`DJiKS;_>)oAH1Y)fG31D z5|QSWZZ*~)0{J?srT|FEd-8(ZDV%9dWsqo5+7C)IP^ks*^(!F;e4YjeBj<=hIw<1x z@jAeh0SvRpcN)wtTR8*RSR|M_5F5QPj><&s-MHpHr4+!|l!PK$6Tgz^?>kFuuMo}s z2e8>wiijRFiHkM|5ATo{iMY(hi00kN)gSW0+zB)=jFp(=eo4BsbI|8&85^>SL2Rt5 zQN{9BuXnN0OIPoc%>76|=*K#Ou9%veJFM` zQH`qAxj;hN(yw6m)7CdAln-c9mhDo*{z#obpR&W4vL<#`D4ed)v8y$C9@Rv;9KLr> ztAniDHs1U(~HU?EOd~357odpXnCdWPO?YQg>3%uGSmuDMMVp$ z{V#y79L&kLPm5uX*v0 zx_dy(+vL{I1o_8b4Y1{J1%w*ry2a9Bx#EA=t=Td>_8JFkova^tB#6H-|A{)U`l?$f z@UY)8`On_7fzhHMk%jh^#H~<|MFF@nZ8L$2>#j-&9~t2I>uMWRoN7jP^#ESW z7B?RNKf-di@w--$%KMuG!8Dl8j-39bD7GzN0B2!t70eDI zBAyU30%@8AJ3x~_PvUiKJ^ksHi_yB20_R$AB%}(d49CUnn;cv1rve)b{KSH?InO4~ zvG#^P`9i=^Wp*|_q7y^EwHi_9!=@_9p}hH?4D!j0>`zFf$8GAIlR|QZvx$#(Z$R;5@A_p*DFQs^p zANGN$V%LM;j2&ngz-$FLD-}fkO-_Ld56jLPV^&9zHS-g^J@;?P!8t*=THgnaa^PG{ zh}31em7d#q>gX!4&IA?;ng}8oKn%DXeEa({f_`1e4nTBZD%1-h?f!CegHT1R3gY6# z!GU3uzn=Id^piPy5}Kw*c&~J5T~1G$;yA7_8&?N4#!bM8l5i(D9dLG_23Rei zuQ8v$-zZC*%AYTGZ1uDv8(h@~rvh|he$bX5iUoQo2f8l3M`BVAsjm0+7S*sm= zrsh_5xJ>^!9)kLF1^7KlFt`pV4J|Z=zY%@gpl&aWxHy6o2wZzkc5)p7BtU!34Q*Fw zK+?QNBP2%Z+dxz)N~_qA3D0ap0u|WPrlhCNam7t>bLd30xZsun!914CfcTMQs7q9G zX-rn)z|?wC+O)y4ceYCgud@;AJ+pM^ufdN>+B%jK`qW{X!}FQD1h{Ni9q{2*;Jnikr`7@*jKc7f0FO z62(e{I>Ic~H1ss<;FhQx_gZK;$22=P{->tbxlD%`s z7%z%4EmHcm%w_#|P0gZFy&ZDc)0(ahq zan9dXIb2Mg7g8;)n06Rio9g>6*7G(=niiUmWO+u=5+E5tm6-9&)J~s8RAN7H5kZ)n zR~LWiNYr{*`?rhy!X!!5@!_$M9P1WKidBc`wPdul*5mg>xb~UW2!v$3C|;?z5j2N7 z^y=`+CkaIBfG()e8K;#AJ{^kJ0qGWsBGm%YODGbW ziYOhWC><==5Ks^hkX{18fI{8MKY3!W^?DhCP=#j2lh1ol8TKNnwK4+LsxPAJ0`G00-&XE8t9HxCMa;Eq_8 z2=h)Jrd@PEHIon=$%at~?CI)o)Bc8dNmOvzjFC4ad(M{{jPfS?OBtFl)zKMTKTSg= zceVS4>TNpdlfWkX=e9c48iD2r8U6pNF#JZM%-V&l$HGX-Z9XO76r8 zUZb97z7Jk`Ui!LSsjODbhio8GIN|Z_^=Sn|cgPx0E)47)O>}wo6$Hl{Z1);SZqu)Q z^azFX!j1p-3BpLGX6&WBLQG6uyHBIfD$&~h1w(1sXKLP7`m&0;3}PW1vat1Yh+*R9 zZ^4-Ej;D%Fh~MzhT~m}z+eGmEnPs&{!vE6dyPhInKiB{F*H0l?v*wtYIt5ALm(94} z6e)W{P~MY^%4Bl~yIKT;$xnInE`e){nMEG9gJ7ko3)*A^0{j_xSrWF*@>Jv-u zhGXjq>GS7oOAx0BvbrEIbt@m2Q=%zbjWoGVMVG8S8CP5I@g7iHtF2vqeGG3{d`m5+ z9p756Xl8WcpjBHu5WH379dj=CfNhz;eYHf!jQaO2P%Ingq@Bfck1IqSJRMEul(vX} zgCGrP*b$g7=vb24FKbI>xl`SSi__AkN?&q8WV>dSfNs79dTfrxZS^jEf z4vkLX+!NNmzrykedelUdO&8E^+y@OzPFpDzd^4*tMvJwP+srJvCQZ}4pq;C)%9|LU z&&rQnhYhNm#1*R{t?Xq}2rx3DZhQm1;wdXcG<0}jnGKcZyI?CVt%6$nV8|b!=MP%> zNFeIqXJh$AO^Sd_3i2lDi{r~N7 z)7g0G|Cm?IzCf!j3zh#$Z8=LZBz`UbTnQ~;h~7DC=Lx^TrsEy1r*rwvq6(t6b^X<@ zV}@lI{F!zHYr|O(k(Kg%NXwa!GD5b!8f4`-Y6Ll z>W0={&$Tl0C+YSsl8c*^1=={A8g$bIAz|HCIdfwB&a}XTs)674hL&JF{tEjs;>{In z>@apQn1G!zI)FSTX@P*!Q#wRi3jB8)60)h{st1>M2Fxy!3632HB6kT(ZtaG6UVx2{ zT|m7^{6AjPcVO~~<$;N+<#AsFFNx}omc^G1AXAL_-@dkoM}2W2zj{eB;OIT!qs_M& z3|pJdc&|OC4wwUG9cQ^tUdpr2mkf@zr#Gh){NHZa(2Y{S8gLXt5a7MGUQ?LKzdoh7 zk|*{k4S3g)g7_8vjpQCiniro zmxW`tsqZ}CQ6b-bZNR=9TKvk`qHEf-P_m7y?rQHW;d+Eo2Qt&k{?*u3Ef>DyUqcL6XWWtae|7Nw6Loh8(vL z!W_JmfWfaWR@9#z^=7$yEyytPy0xEq{Qi0PLsK4=x7k^?k_<-m*CUOayTYSX@aCz= zKMMT@Y~*I`H~Bq6NRgN9*}9{JfLZ!$1^9bt-JJ$=N-mdXc{i^Y`-ocwj#n+3h0IHfVKA1r(PmCI&fR*NU$>37N=Sccs!+=2cHNxg^gC;(rE?QniUL;S z+VjWV(%mho&X_XC*B2`d2AB_-kL1?Def4(qX3x#v_14Q4H>KVB5=ZWq5`6+wW@v5# zGp$XBQdbv_fAT!l*91OT@pmV4sY#yU8?}8Pvffy6hVl z8(TrH3pEse9AYG9U)~Elvle4g$4d%4CRS@g9S5JMOzlO{`S$DPO~c)H?^T)(FI;m3 zbj&1;{gT4B;?b`wYMOQ{zts9iYyGfsovOuZZeD@Qb7_xAgY9bT7|JS3-mK5LP33F< z=kQD@Te{xIBQ+8c#;hf&!JH=k31hB*OyW!UgigwN^j6S;Mb`N2#^^fkNSM9hFyD=n zQsJ_gJ^DShGFakKCFo+@{&w*)T^`&UszToS9~E zozjc5yWW%{pn`ZQl7sBXIY`V^4(2C`*6p4j`RMlXxcVn%C73Gcza9ydtm~d1nS_*` zC2wgzGiPdIPmlpfOfA=kXrzGtCt@d-*+M&hR*{JNHxpNn4UTPn9hpn#A&4mn&#%okZtd8@=*?+YZnkcIo-qME|5FLl$HWB=yq+B$^Pp+2udzm2 zpwkxF=<#}lVz4-)c3Pol|Cja?jcW42L4kIOLWTR9B+;C%;zsX-mT&JY8TAs`$s_eP z?FZ7vPCbrRmM#BXM2E^J914?8M;3#s&z244fWuGNnp{J`(EV-^0WOQ8I=fK0@rD`| zuv48ctenFZr#buXhFK@!&_=sbqtVz0QZ*qdd_rraaE7%?zocrnJON@Zp>W1r-Q2-M zcp4^>JyTs-Z4E^o}3$n@x|N?1j5m2-umzYw}eZ@6rg zxMVe%sXgjnoGvt zdDDJrIvS6z>g+VYHXq-8N{~}d&`vh>=QS;~sK_n@Mu6IZ(3jUPn_&50zj;|*_|_qH z+wZgN!{TpqLTS)(8z_rVzWg58K6;@it==}06i^cE2UX!5ry#(-IUWj#Z!bl5HH0~Q zt)0(A^)`j9&4@CYuD4^UYt=Y4MXW1ArKYI^6J|a?-jie&$CkUx^oUu?v18($_hLHP zVocm?>aWEYEN&GmMvG==|GXBEOGG)A{JdIdTPYR#>b9WMrb-jEx)-`BFx5$Vm&&pD z+Z9>X%TsCBRW&N&*dm=aHCxy1UaV@vlJK!P4e`qEx^_~^#jo(Dv(htf zanF(Hxg?4BNGZjQg*(A{RI7#Dwc7F&R@SGjLhF|=BLVQwx<*F}^>p*)D4KRZVM0a= zD3`7W3(s!GIoH4Q9BOd<^8AoPPgtTm8S52%5jTE}`|U3+-zno4rq)gb(QKIgn7B*t zsjMVU#)fhs8?D}U&r{H7AG#@;YrE-(d=<8|&N;Eg-zN!Z7n6cjx?mPm?Fz|(hTKwt zt1Q}>|C0`uj6)mT1{ygVmEkh#Pxr#by;W7k6S+|Ol2INpt}O|-BOrND=?I&tA3CqC zKpHi8|9>A=Q<>C9$0kL|hEjvSNJujgPPCEwypnqoWUKqWvdYu<1Q#A0H=NruPLn4Q*v-iQU?rb}yIGJ>tS}^B)d59sctMztcgIRz*j?d-tW*`zLgoDwm!Ft-Tbqt5jU7H)Lj%QPw`UM%1ZB>HL50XWchCVkBae zRjRHRsyxn|4Q#B&Ms0x~#(BP4*?f1bwo}u)Y4I_&S&|4^HI_?nHY;V<(!5G%bX8kB zq1*%xfeKZmR@g(H!Ui2D;ynyk389sMi5z2IYTe(ZLyKIcIQG9A5I2jU3q)-Ea&2x$ogn{Y$Gc4h zdsco+$qn1fY~Hq;DAeJ9EbqtYul$nj;a2%(75$=a+F9`h@v?iVi_<~e_iOjN z*F)M;nnr^CQ+)_50tED5DiMMzO17;hdN^gR-ybV>OTYV3=jT8-Z}jc)9|t@*Orhtj z9%dXiZx~GOmWVPt^roRRGDl{|tZ5{6hbdpIWlz0$T7t)g9%Rfax2N;nbX&0W@inrhWI&dbx;W>*=t&_0~1@u6OH8wuq5}_rG%0hYcfU?^aQ(eXqar zyxMg?M(~xZII?$W-`pFv-HI_f`pY%%b%4WB9+fX5s^bN@yt_O$<@3Y`tHeoJ?k#+w z=HbMw%R7QpY1Z7zo!-=Cs5wJ*vX^bC14GMcsft@Bm%F}QJoga3qsQ`6+C<{Thq`ZW zIVuK-boPv=K4#AO_pq@|G*fx=R&P1dRa0FJlJgrmMyZ%cs!>ZCFg2 zu}Fw&*V&LlcGuCnX!I*CWfP-|cMlS)kDVI5t1ZH*SHku0Rb07z!s6x$tII|58&})0 zMGV}2aIx^eqJJJyinh4yJ74g58Zoe7BDjPDratoySO_qoT6&C&ocY-c8kBTAgzv3v zpA}Tj8YK*}{~agU(+2A&;SML-hbr<`qJ<6^U{u)|UoK6_eKhXi!ZLTn5&OEjsKWi?c!2QLoG4dG)n;gC$QK8H$0JgDTpzgQ5*FDLoxTd9>l^NpnKVDYMOORr zCr&ljHpL$An4x|R3QPpElIa?TnMqIVO%z`Retcq$=0ibi4fJX&sMvCt^im(1hbc$6V(5M4LR{9CCM zCShDR)OHf!U`sjprMpQ#nsCh9rq`b-UCu+6=_)MHI!=G@%#lB{Q*Gzc!R!3471`5+ zGmAGDt&4Tigi{1fCqLlU47h)ogy!uX4IKRcS?jK!8TwP}lGsAV+toaF**1uWtclg- z@Y7g}xZ1*~2}xTTEqo-?C%P}-s8H;5FFoaKU&~VPw~lDznh!vT>LOP4U4uP&P9-}o zByqR+W!{o&yQ=rGdJ=ekw+C|>nda;GaAyZ_*U$9`O%H^1mj!-nS_~!@^p!rCO@8BE z_Pz$s zT$1xw>Rq;>!unHi#936fpgWg}-le?i&l1|yL4N5%zbSD_3n`W|QZvDMVZG1!*`1yl z7y2WBW`AGFniRB{e?zutSM}_vcHyULyMw22MtuUjpcxdI@;99-*P|H+sK?HAv*;~{ z69G9+jamwFE*M~}t%7ag+ZJyxfwsa61*=c`6m?4`juxsRTj-*={lW(FFuTQ zNeqE;WY@{Ko!Mry+&YRoe~qYDd_|xkw^e+4Opm|1E&Y+dK2S|)(YDwV$Mcl3zN|HO z2aif1=bF1zo1i}zp~a|ATjjU%MHTY?DY_2dk4YG{_uC)JWf#ZudPO$gqLWGRsC4%m z2xN+Dv>RLSj?<*b?H;ow;%}Xnv)ix7u;;@aeH%{6X8)))odz3M53psX-Q9 z?(kH3pB48!!#v&p%Lq6sN{Z+KT|$C06HQRjUt1mTxPzO^p7=1Saj2#8A$iPsbmjcA zAG9P+&Wph@?Vf;|1^V3kly7G9Ta|cw?$d+CEse95w~_XHRrl$` z_Zpk_WgVYrqgP>Amu(H{%U{f!a#=ned>yzjVh@;RQsRen7l;)PcK^hmUgPIg-VwfJz^RE6kos@<_k2OLGkN+04HR3H;|-RGkWL5iLT z9z=(R5U$k;!}UEq$bn1!0a;I~cHyqzm$vvS3F0{TWj4-3$Y99)_NgScAWLP4+L5&}t3BnRp{n%IvTne=-fDVLkmV42C_XSlcD&6{t*0SZ_FNw>jwyQ1ZXEx$ZNGK! zK>FSC%J6Vktb~<9XYgCRe4d?QVIbx9{n*>Tp?NB|Gx+H@%NoPR>eab!LbO4wpW=_3 z_X&mSw{*GFJL$FcsqPqIi=K{ArTo1u9y!r8t)Z4vxe}QcJj}p$v)6M2(r2s-nzRnl z-dK-rLi@zyWlTO>1>tv^qW4E_6HY2ge0lbMi~#qmmmAfAc1sa4F6Q%Y1Ctk~i@ux` z-U*Hku8H%Xvx53x5;RmUquPJo-PI_XAlS$pQr;ggi5g+~XDqcLa~BzAX1JWVgx;jb ziiz9v##`g_Bh5HFMVS3Z#@^f3<=D$u79GJf@ocq^i4b|pjbbgSb8P!d_4|IoGnPX9 z<84eo5;WG^PCw$Xi)KA6YY^xV$PYGg*e784$@kh$;Y=oSIul<#y41GMFnru*O>*a( zqq&|?L-$kZTq$e)AG!ze9g*Z9B3ut*3fu(9mUQ zMr4GU)AMXFdTd*M$^)J9yE^5n^~`=nwfdhpwqy`~^;Lw>**l$T5^L{Al(~P2t~2ih zd`0{3;$IV1)0&dBk<8~V=I>tnHSoGg%%hRXuFc=>s+YxYMz5hy6K)Ap-&l{)DOah-Cvobro5x9^fI=awrYulwH9XgkvHAbWMyq8 zRr-fanMYyWF3NuP_(vSK-%mK+#@G{-dgUB{yz8sKs$@GG z1Z+PM{XgCGD|)e&oTB!`3+3SW^UHeMy;Nsy^EW`8B8*4dVVAD9#(!HJNlje}%aQf% zM0|MJS(h&t$chJTdRSv<9R5$>;`bada-|HpQM=4vb>2(O5KhbOE;$m?+K~FvnPbAs z&SptOgwpSA{{pmthz?k)(s&Zr?U!rX;l@4HS?V4O`Ms?)TJY*Z`5_hQJQr&b62lyU zwsbg~YWG<8E#KeNl3^L*rmfZ$Yu>Y(O#hs^LddpLKB0^akg~waPn?NMQ7VaSjf!$( zj&_KzzM;NvCE2f^Nc8=ENr#~sF!;_hS5F7eWn?mGs=ZD#HTk!@AbLLk1zl=~$)>l71!`NwHD3QmyT19h(T|2>D zww3&wx?qzbmhv&kUm}j!bXJD^qBSBGyAv$;_2jeM{*4V`Y+>xm5|{JTahSJWa6FUT z_j>k2*1d!)NL{PV7K7+;fU0dfx6pgyCiHaU!m)>j@kzS=FrBqOuU3<-GxydLpoN@? z3sA8w+J(DZXpOEU_wDW6=Ss3+dOEv36y8%<6C|fB2D}i@D3}J-*`G8AD&@9$;k%W!Hx^H|e64f_^lq z)U5>z*%d1$mjAZPSlw0~Nkh|*oP6El^3C+ z`={%ycLF(}IerF7J~z1y_RKJHkn3w086~=|1hsz{STy=zs<>CRHAE?gB_*YWmNPBm((CfJt-%!zBhbgPqADF5CARTd73Pc?+S}be0;VuU zsm1RIzyn(Q#mX<6d;)*o-S#`&YLf=(tv52EMLD(L2=)^N6x1fHpiV%@inbJ~S#N92 z>W=q;BvxXqB>JqpCT`nV`N%@Hbc)^5WA&rY@26a6iD5Uophrt?2419wiEqbW&J~~}*!R8hgR)@Fxi2ts` zBw#6z>|xtB3s+KA>pffZ57p;ERT@c+Q-T4PwI{bf1dfYRfGlSNezSkc#r~toEv!GLA3lVkrRo}v9RY1^+)TajfGqL)X-s6v-5mD0g}KYDe}yn z2tm6Q``mdkHe>wE#eCc#Vm$?cf9SKNM-|HGEB@Qx{bC@kVrD2Jm?d@1H}5TXo}%b> z@yC9jI26AV@A*6b`1#R3e_V3cPbW3JS7XP?R(Kau773NcXQLT=i_v`DfB#NYC<}>Q zh8iK#&H=ZiF8^jnljTVI(%Jgbo&A%1h~}23Vbn;u9nnnk-srP7Ye0YJ*|5xVJUzUy z>Ye*Zs;6F!u=wZHT7N)iwwf`+?8Y-> z1vw-NhWxvRjaqnO*rsY+X{uOylYGXh+t0AL+g>B6Nb|(q>0jAf5|rL8@5#E7nZwzEz??; z57)C;vq>For?X?BPC4A?(s6+9u^QsKF1G~Gx6Bvk_&hp{4rZ^Y?V~eRAdv(pJWy8O zIgF&!@1`ouO=oR0-Zrb!nPaSE7UMyz82*ft(xREmq;8{fFs=k?FRz>5)ty+>V|h3S zJ|A4ZXjR}ZgH}|zl(%gOmg{J`dX5CP!b=Pqfq@A6gL;|>d6~DC%WYHu2ifUK$$f*hQ!ny%*uSEqw6Mv{Fw%7 zTFJ!^kIdLEwl>7&|G=Ly`?RNpWO2-r6`gVApNiLkw6j8NC;>V>KE!pQ=ANA*c0j4N^In+p|Vnmz0q!FdF3adUX%vi>~{@SwXo}ysr+ZS^cN=^9*jy{q9UyeV{!Y|5Rm5uMSz^HulT-7a>1v6hquou&r^=TUS zPpF_7)~zM_-#AN~dg6NzHva-lhf4)Zs5Iec`4^qW!Jr5ER zP{JWAsg|Aaw?LS@nuc~<&ipL&r=^AsWgF8Q2v&WI|8_4Gz^d9D!G9KFcuQ$Ma7|+u zYntk4zax-6mC-jpNKeF9L3i`N7uROkgalOVFhZ-IBk8$@P5UyTuYel(-=D&D0RBcY z()}f<#TlF)ec)IddqWfShy6**k(ox70#bpcc=o^YrGR#LGs>fSpellIoEo!yaID zEg?)tnm+R+)|FWYv8ew>UxA?q$FGZxC7WBIN5r#&Fky{TZN+MJn>!m*)zm<8F((Gh zy9cZyWIWr)$!G3p7FJ9@9TLdfKjawYWeJ?4f3FK*gm7I(7V8egmFkddZH&3w&_7m) zD7XSWSmUM?mCVw<;4@uBCyX~x=zVu|VL(d6QkOt@-FO*w!oR=p|GqD0X;ZcQ9+!fl z-KBNo^{t2cnf6~{z-=M~u~yjCPoD`~c>Cw~KOL={@1?ZU((QvFU^r(BUtO`3%`bW2 z20fNqD7)@K#H~?sL;7L}EV0nYRWUe^Xa(7yw_(ERXObE*&gnJ_yh)Q$gazrA5 zNYB^z+f=CIp&ey@moeHj#U!whpMqe$sT_cve0Ljx1N83SSmeM6z}ssB!D|cKHfLa> zhUiLg^IPf2nBdL9{nIo#g&`GFAwQ0A z=1v>wAon{rfBgh@txv7U$tU29=S9OLOpb+}w-FOR@*$`i}5D?E~}hWR2%Xx?r- zX`~1;p^_nPZf@<(&6gF=Nt8}QA)#oS-@>8zc1J6#nVu_LN3f_lJFqnN!(vcl0;qIq=4rdiO zT~LgIS%S~>m)ovpq!rAiPcN(XZ01zHy!ez`Qp%$qd^c`o*v5aWSeHp9+>?D`*hrX8 zn$36g3*Ib0vE2+GY{I&Ouj86{_#LR4zjWR)rS4`u=Gt zx4h2X<4yh9?t_i7?kf8BlDszrt0#Sen$d=lVJO^&_@n|v2H;IYPe185M!Uwav(zJX z=yQ9b7tBb~ixDQ!16Lk?DeXfJH+#pe5RagO9YJa47Eqb7wohIDJXN97nM)FdC(D6J zDD5E)!a=hvZ(%umw%Z7kyLHKU`C@m77O*fPh8UfV{7Mf@*GD4wS|8?2lMOgcHfNvR zX_UsX7Ig)ask(XHYj(m;C?c4&Hb;-D^AzUq$#Pt2xOw?zw)Jiy)z>?&iH0ev{902E zBS?f3EOU-aYz~~$TN^-@?d@pmCJFUB8yO5yFBGSb& z9H6NX{`$0Ge0@$n+_X8_4!0xJS^4zdJe9V5tMxv|X1Y}-V!mFgf0o`7uf@R74rk*E zYCRsbb2E`N%RxQEvRvvQ7{;Ww<8K}RlyvVVYVKx8y*J43buQdmE=?jygAtLXGG@H* ze^q@Nw13S;Kls0g+>+_u&vFMbR*SD<%N{0R_YoiK@ynWSq9B+NVnY_|e9xEbJMS$i zunXfO^;q}uk1&SL&T`ErlX6@o!fxV2#fC!8cz}M#;iiOfN36;p5;QAI71Ow2Lnoev zO|xZseeH$59T>U5r2KBDp)kM7I}r#zdK1Avyqq!L9u4=RZK;-qF;?vLwO5|^O*8tq zDuOBM?ksVplpPt(X(&H}%mf_*lL}=JuAsdGH;;PBi&kUVPyg@68rm_{`}yfq8iCOl z__YC!t5zvU{+qIaBRj|L2|A7N@FSt%{caGc?1FxPDOmO?X0j!Sv;H;`KX>E$ZF(3* zt;q{EV%_X|gpJB^i`V~tkC^?k>HX_Y|9pz(xjQ0@&o6D5ztzQI>1CfqD8f;N|996v zJCcS=pPg+-7LU`Py^wtoFD`IW2`<$~gEe+dJ{ zC|>KE8pHc_0IgH{j#F;!tm+2G&hs^ZqLG)k0lrUCl6L7@FFD!5%no;wKTmsQE;Xu4~0#t=lC``|9bTbzZ2qmR>fJ1 zv4g}mgK%7;dr zY(bp;heU3oA@N&EA25_38@Y}Wi*!%RH*1br$MlY^43j*%Q@4f zZq1=$OY)PgoHUHtqB8fxhM)Gwh;yP`>dVG=sQy>yJl5wt29O9Qs#!jlb*1X=8`t(O zgJt7Ft5bq$XFLZV4}JEm6S#A!3Q=8GgCJm(IMjJ9zz$52`jk(?SN3mTg#}|EucT!m z%h@Tx&}3(GoNu!yJYsP;KX6b;TQs>kExgH4ft3rJ=I${MyE$tjz*BCZp}4<{?uvX> z13dA?CKkL5?B9KI&~5XjG-Mw)re16aTO-LEa?}y)AZRUvs+xg1dEl zIm;KDjX<~{E8abQ)%Y^?e$0~V4>Wqa<~5VZ`S)_b3>96OcDgHv|_OLtLj z=WXHtqD4ZRM=~?Q27HENrv17=&4E|iUSHnKULPkJK2dEfU-_OPm%%SD;_|bv7Y#XB z>AyO5D8+eW3&(UqnQK*WIlZ>iG}OoeGl@iXzY#C_;N!dBUV*&K9#eZ?sL~pvRpYyC zXU-I{1~G;z4jMMb5KlYj4JS1P!x}?E%y4P-Dq%=TjUjA|)ZAP?kbObg*9zk!h#5>oFS#v8b*nFoN> zty^K|I9|_IHQ62sE0{EPbf-iGCAtL~rCox9;RqitEaZ3vsy(A|Y7~hbG4g;J<8#B)L4lelFTS?Qw;hLHac7`>v zptkz!oudd<6dMeJYT-W@u+_AfR3H~*`4<}cX()@uNx=%wnR+4 z;YcpZTPEM4utd2tk>{ua9OFL@xXft=fa(Yax6?>EZpwP&vqhm|b)*^X5=J%gM0aBq z^}LQCtRk=)k8_ezqGTC1NqDLw_PNQ-A52F^mqDRVg)7(O?6|?|eg?bQCS2e>oJ|js zf$=MF`N7wfT08y{3^K;9>4N}uPRH08|)X4_)`xw-a9RI|N@Q?qY{Xp!+AtBDr{0R`?+ zJ4wz|2@1>hIix|_1ofFoX!v1M^1BDlF6@7IcAgN{ zcEg;La|1^4=k#em2a`$2Rn+&0#`HvUx6{ER{C_@0^^r8;jMDa9R%quH1CJ{S`DTgR zs*4B+Pk4jJQnujT^*)2$Q8Z7A{S`jUvFmtyW5Mq(62D7BI%7S!ww*LPvO-M6T>3~h{b`V*4Vt6hcs0rFB3;WSa5`e$Dp zCB{(IX%SJ$n}z|4QM{57E>omDikPu+iD$r?JdpPTa$uV&+^%CtO);o(X;6#cPK_?v zUoqsaI`CBD1I{Ph=l3^tLV}OVICLHVXdNv>1mDrjjVR7WW?~PLeAxKS(xe}l`6Bis zcwLN$k4yfP0#(KB9jPQA>*JFj=h@i`YGc3rCIm>Fp=)JPu7>Xn7FHU{+LvL4n+-Qm ziQD|==g=zxe>Wmz!)U~LFl`Nz>-9xsyEHW6O>NE8;-q{of42(TESyvSjsH1|$4d_} z@VqFS-yn$~>F!{9VC zH6x0w3+3%7=h#c2E7whAOR1{5mkWt_1vMv_mcp}zs!9q*9Dc|N6blqcIXL_I>Z8;Z zdiWxr;3T8h2IAS(IEa~b789gfYRj;YVda>G2j+|KPnPaXd$pjJU9>u*wqCp%y52iq z#fzZkQji^tOY;*1D*mOTo%~^ZCilPk(67vxw}@}R3%uDK{wJLvG-V7e)69gEH)_2i z5kz7HE05>36X6_{Zzk5l_MOw%0L-X)_6=Bo0m+clP%#D!jW_IubcCrbbBELihwAP* zCxQ|wg8Z=u#DvNYKWjiNH#W4HSa$1{g$Sa&rQti2y`khH&oiusZ-0s@eiV3AZG-y! z&$@VQ#0CxP>0QLH^fPSF247s}W{blLdE9X}P`^^Y&)-bV{oen`zDkGK(&&3(X;1=B z^qubi7Ki`cBtaRzaTZ0qzlxlW+?QW!!l&Le_QN|R_?yzY$?^&a-Jnf_DA&g^df>kx z)1U7|=(mR`#~l?QgDN<(KHS}W@~*0WQ^Lu*xUfK!ODD{BP^OlJjT}e`k>Q&>Q;`y^ zKeHoEwSu9-T${^3yw0a`Y{nzA&0r1FZN3cOR)NC*)_O!i7e-;Pv3o}H$xC1>{e4U= zXs768M}L@UQ1Tnp)bNqN?O;%vegTe7Dwu~X7GIN-J91h|R$2dRb(yA>d1t+QtVJkl zD8r2ePhs=aLl}`IpyYkDtHRIg|4P^wvN|5KdzK8+TWW<5b{WKGgF7E+vh9qSb^jsz zq~LYlc`dDw^=enp%Kv>O!?dBl$&hO6&pXNhCJ+$44b+Bn>rm*UXAUGP9`Y)|5R=Y>t_E;=b2x)jJ z{@;oP%gfB)DD2y;)R?8v=^65sJN!hnrG`AmH+MaaA--jp=Inf;!(3BW35aPPqMef7~|nw1CBcmvcx}UtBFy>K{O5c4Too ze&-m6eX(QJZ9axCqqL*fX&~6y#w{l5Y!j`?cp`zW(y~-V=-pX~lW{AHOwh$95R(g> z_OEb=P_j=fD7)c;ma62`+|&|By02ON+g^Zo4>1wxjJM%tq^)M5IyD7n9g{$rQgG78 zUuFOv8?!kE7LBKHVBHUkluAnX#X|Y@*IwV8c)XS2aIH-J9`qXktC^7G{cP)^r!SOg z+AU63@L{xn^~9Y(hNQ7MX-nWD^QVkRYQ_b$L^~<=zsX4FeHsZ(vg%sJ#gg|V2Rg=} z)kLfnlR8ogQ!{%{qLW!E4>sMb{v)7_c&aLA<0 zY-=h`eHiu@S1eqalv(49TNa$`rvQ-q+3SDKYAE&N$TcGPz7*#yRO~$)c08&5W0O;o zE7u2h$n(mkG#J$6*9nCv2g4AwTUSCQy`kb!*hv-SBtaU~ zE$P=Y8M1>mB_cpe&woUjoj8T-@Woa+w)~xcs$HR~2h{ihQU-lvkVWh0D{z&(bt4Ip zcA1|3Wd4*-4YRD@y$+74eTMQSmv&L|Sb=8*4lAdYwC>OEO^`-E*9`{e^lHREJ}I#V?HxVmCrD{ZB8jC zTJ_|{QBy|Ad|}a4U0yj@RpZ+KeB&ZDQN%AP!AqmLfMb$x#$AraDh#sULkz=hHZCw& zEQut+QQn?>+Uq(rn#Sqh&P)(H0u8$L0o@tQ-KkWj4Dq`JRQg<|dGM7z7>!z7ihskL zWW%SpyX>`o%um;zn===0E7bzI#;xshh6@WPmckP>QnZ}m!MBD%VnG*ZrYurHkj;rN zuX0EuNm=&dH}l-Q1wf#k^VQBAoW2&`3`dTCFvdti&=-IFF|+%LXHdt-55EDA$Y&^z zV-YJO^9EDn@puVJ4|K%(^Ku68j6@>Wd4T;QFLu(c0Zo2Bi3D~5>@EZt(7g9O49=3K z=c?0@-f~$)yTtL%Shu6Qt~G$VbHEVJMD9rDz0n<_RT&7hh&?Jj)JDeXE&-?n(kV|} ztHC?q>bW_3mv;wE@wN&yc5EpZG|C4Tk1m+&Ime#5IkN}mMp-}@_RXK2d@3wT?nJ_S9ZNq4SFqfpuX z10i$Blr`i^Z0hzIDqrzQsLx9O3`+E?;^W#D8A_K$Q0l?kL*t&ar6Z&g_1rsD33k^Z zzQW^nOVbvVDn$NO4L@CV{o-8`G3mM7FTc^P+3i)TZkO6UzPHo+s#W9}qCSs%j$CTp{@9F!u?NBKNhd_pLtlarmoP=yamZ=4m znz#GX4s|7}OTgyihF-&8pTuWKLdtJ9*LM9#+A9M3%AwgIp-8X1TLLN~Rw;4O_sWs_ zdGzg2EtI$ed96s>2u|2jN^5hZy3G4RmG(Uq52bl;z!Arg+t9aaYzW{YQ>IgS=$re~ zLV(n;kqJR=%9t|_Qu*MDMwzZg#-`l8qH>>tI4Q1{hS>%dAHdSA!_}I znn&bnW(OG{YW;a%*q-|LA6dCNDT@hmdSIBj>%fU4upUo|1f2Ei{=BAF z{HpE>zD!MJib$s>uZwqu(3y0YQyU&EH)dU)qs_j3I3U?7pelQ#2aBIl8-1(Sz*5>Z zcb$Laq|mQZdF+2b8i<0>fOHDr2ISiBZy5+do~nyFTUE{G^Oh{AXam5bpY*1}{zmAP zIiNf{b%DDAW0Rgkjra{b1-CNVD1PYGgzmDL!1WWC5g-jXVlKsM)Xlod5y(mcb`d*1 zI!L&9+GU(TwsqEbyq1r1Zkhvf>z=U1I?*l+iO-Vh)*ro^PgHJgh{Z;#!I^6+Uo#Qc zd%~fwDhP1Ehfvt_>?R2Hu8LR@h0GJ`jyL#@#n>HllXq-R^TofYdR!z>DBnKh^gY99 zlXcMDCBMaZCr@RJ8_vOClTx2MWJdjBnSe!*9s^c{GA{4#;q@kZ$SO1SJlHn&_blJP z+ma`ieEDui7J&OeAhL96Dt%c{y|yH3)ArE8BxM)gWwoRARNk+}Gh-HZT-k%PSEdI> zXH`Q4l5fpT($!WMNAoD!S&=%qu6@hB^S=eJKw>>6PAPGXNs3KzdA7%QopoMX?pv>-CNq4Iu`c%c2R#9HdYvIm<$fD? zTK2{%LuolPwKvSSz~{2cVP>4bD{qJl*D-MSYp}`T4691(uJ||$rRun`mU5wnPaqHW z9MPgd5uQ96p0R&G9KUn>x~KlWt>3V^_D+nQZ$;{5pQWy^j95k$u8!)e@S|0zMKBu_ znK(KkI3KbriH~(A&S1rm1IhebPGUatJcB;1L`s7p+*ecYgDRxf)_}Yd%tYqN79OB5 zS}FTjV=o`Du_`QncwxlQ@;c|=8{bb97X@MDOP`44YQ`KhzWe3YK&=A zC@Bpg0qjv%PBZ+PUXARSmh|7Nch?cT1Ja0!S?YqLQP**w-*0hhxJCScS!MEEX3@f= z37-;$f^&dE_KTd7FU)$B=0!Vmo0_f`M1q%J;?Z2by|2sHK%@s_{wUYW^xU>7@`dDb zwvu}DwCLM571=+a{}V4BKlk~VW$t;^cK*~p7&`6vf=Ur{9y6g{5wUuN0PU*(svh9_Ez`k#}i88rAW6Zl? z%|WR3HpHRb_bRt7EuE#UA>emkcP<2FlHv?wUaKv8akNy+aqnK->gRREecomLE0{OS zRyg~I4(;-jNy4~W7N0I(^_9*hjqrmxUQGL=JrCJBl~-j)PLttukm&(4^Vm)g*R#^G zlG=*TuUH^RANx_(<~gNKC_T@S>^Gbc zSBXZa1je)$gwqc9LT@c^(@)RLxFW^@9S1We^=)bHHQFa< zYSQluI)#A$@a!Hea}nq>r?t~CG>TK;YFtO#%FW6^rylE}IWM#6CUp~4hxV1LLR#W9 z7LeFH`a-aU&#DZ7SsV8Y-=@V&HTvbvp8Qj8&vLchdLJ?jk?`wX>Z(ji+yA)dx{EfP zbG)B#|K2JyqbPs*L#VI2(`2bt!fa_?^(Twcwd#LuSPh?&CWMH6Z*=}Iww4q|(*jm? zIpUs+!{_JoLzcf^KCu-B9Fipq?WVJW!UFq1Mwog%?EU7gft22_+ld_&gUA1g&-<;N z7;oJH6UDwBFUlUg6&T!FIjMfFYFE%fT4eE#IJZT$MAdbwp#RedJKZ z_28aQMNF?uze8$GH?8bsS9UF}y<8zLz9R)6bqk1HwC>%aIZpQ`_K8$(MHp@G8k#bxot)%W(CC)C0W zkIVL^_);R^1uYW=?r2yYCHb~3yhixmg5Lu}lr#g3%2r?1Bdm!!mmw_&gYGCC5F|gl zl<3*a<0b843kwRq6|_A5>##|Ai!`ruQ0{nOvZ_|UP1^7Y_aE;?yL*u-?6?SV7*Lip zde$=4*ygZMzWM9{DidzLXzCRJ=eJsAJF9pk)0uvDBSWUTe0sZ+jK{ zPLScD>2^#-02az4r-IB^m!9UZI&EQ$gEH<#1Ag%r*&u|b#FfB~du-uDwl*Ud=cDOY$bum&Q@&1%i2VaF`Mur8gyZ%1zjk@@(2@eCOzF7gfZu^< zgzV+J{p)FEM<`=GVmJ7AHEPYn+z3#6{J9Z<{@{(j^G`G)dHUidSliAcm<;2uM z3fD8~!JrExkl%F>IbdxGIwUqkDv#8;0dbK^pVSsnQE5v=j*pHJFt0JB$!(oN+)bkc~R>Zr835opb7+oxeGc)WdE7ZlcK6)7ho~`L*iyRh^YlOZ3it6j461 zCS#SZ4{^>~q{5JwVGdARgri#aiW+^4;ic58*hc^{7F)wuvaSm|`@L7_Z=59u^vK37S#?01b1}`>9vaz#q`uOLgsPSgl*j264{7XGxf3%xV>A0L zpp$jed*vwe=sfCGe!O>TS|(myWxM;|9qxf!kn0su5=pGV6Tx!(uXYL3RC{9OODxz^ zJhE2lMXA_2CNMaSi#qSbw4eKQ|JGl)Z4eK11)HaGB-h;`JwB^XQ0kG@Ouhz7iY-5_ z2qSRwQb>Nl;1)?%82*Y#7%}h~1F_0vg8pRU&j6(Qp|XajGRZ%1pD75ln^k4u(V(K7@?5jO{Zt>GEo+YNmuKeJfjg0v ztn-(~sn+2LG~1iP+*`_mPX=mkc>MC7{!%B&O{%I3)n{|$FF9U@4@uUB!W$6jY!!7& zRw+D^`Kf7AlaGA0zgq#!!SitcpRhd2|RYfK=u;@;jJW#}TvT zQ<3K4Bl50+tr5@eTOOK)9Hs|uR!x336h$&U9$7Lj`4fpmEmCM%4-!;SjQT8o(K&rw zuc??E$M+)@ST0($*0}dAf2b&L7yB{Ooa+_!i6wC|I4q+kR7sLX6jk%X?Tc>29xx`Y zb+aUCS9@IJ$X{oO|3C=8;OkH!B*fv}wcx@25m!q;O$2O+2G)8|h_E{4o(VeGqi zE~A*U6Sab7;?XOL+#~$OVeNcwj8(#7xa%WPP_6D|?;YBja{2STj~cCgaj)*SZ|spE zH_X~_n(5Bd&E7m7Jdk&D1+33x^u-Wr)UYRxi4;?Y1=b9;sDoQnQF}JSa1<#OiY6>( zdv}xyYmT-ebW884gS86^#qkZ;~CnsuXxmu+T(2W9_Dr0v`YOmGTXeRY(^Aw4I< z;pm@k)T7BsC?Yz?Dcw zCHgOr9?{Y?iQP;rTCdz7CH_=22@1b=eK9g;!0O)nZUnE%btl;Dm)DRSyt2)DsJz$V zSTyl?0KBjswi%}+*AkohVlMctCKa)(zNp@u2@a2S?~2v^#~;rrX@74={@Af&uL%QR z)A4#$I5>*HeC?)=YHa}&z%gx9OXvP`f~i|<4A%5uLx{Z7gg*fJ=XiFgJZ-ivgJsyk zy8~^Sk0=i1O*bd~{q*ji?v)$mY5xE@s8Coz!hiAH_p`eq?faA6`=rRf8OEr%)39hp z>8^g(4m_l)izVXl1?HGMSOLHG+H7EHa{T+Jd-nmW2xYGJo#g62Z-<(A4Id-!CRoJ& z^Ka5hH+9(Fr5d^C1!VsJm=}s(fJFi<(k%*I zmwuT;%9k3lL;_$4}z;Dg!B2b!mk=I1TtJ}A&N(-Du8aM@Z;6T|GHvf-Y}$aY>ImA zx>C46RKYojUkXA@oUaKA-T>sZ6WYxLNSf9?S6HR{j9J39BZwshnzn&_n7gj&r6BPC zpabf50t9F$JNzHen8?`!3E3E21(@}KzFrRG>_mZfk%r&MrXUiIhgA?A!Rym5)?^Y| zmiI@*zB-^diZwH}2APc zXhtPNG{oPZxRZcdKLdW>Jt;6svzpU6T110vYYQ=3%dRTwkvS7cY}BM6azU;4VKFpU z)YmN*34*nwQQ92R;R^!Lv3poEu_*_y*h-w`kl zDBxi_W+1H#r=NlST%rDsh73P!<^m@eCVQS&8PaVH;JoNyt=OYmFE(cfLF8KPf)MSi z4qLjTeVa+--ZN`pe`_790JeZF`Wr|R+VMC+Sn3GFow_T~`mgVs!wg)PIVks$$!ket zM;A7^B*$r~t3|3{ot`ly!uSoW{_rXxJvwh%9rTrZlx}4-f@nizNPUcG+C+)NF{Lt zeZ;@&(mxjn;I5acVnIYf3K0YjG;?M2NunzM&E0{2MAewv}59oP@T zZQfB7nca z_jf!cHa9w$Xwah5hyQaepTM<5_NGDb;mfnR=}35bU#TN$!(}V>k_Tki?uQhOuox4G zv9wm;G?^WMg_Ynn$TmK2NGv12`75&tHI)uEB~%qDJpX)0pW#CW*4s-XY*g2bd;@SM z4(Iy01x%#DRiuA?@LEw*Gi)MD)5wSg&9njNP|_voOJNBz&2jJ$&Y8d%)5q2Lf9jD< zA~HRxtE~dLzgazO4*nMGzW&Fe8&-+oIS7TM=(vs5lZw%6jts2=+KOjGXk?~4|6POQ zAO_ebF}L*|$#muChw%q(d6*xPNw$w{{pW{9R6g+5EnjzgQbPX89pwz`=x`;goU`6t z|1Q$b|GEX!D0|H5>;77K+E)CIw6Ycpe1_3B@^t?DY|=o2*|%l*=icQveb@s#qPf|O z%aA>hAL)O%gj~JeUYH3}d+N+pAP7DJy z)Y97*Prkfyo|X?jLfjrw7eFN{(tZE#m{t%t=RJEdoS{~x>&N*sH9QnCW#0PhNQ_E+ z=2soqCkHA zDsg9{OpKl+Kn_HZnb9>t@m48uzrmi^GF>~zuoI4{xpGEHcfsL=&TNAUe%ao?Wnyz3 zvz$DE&^<&^q4kJfJYFI>5T}BO0Z(2gz}njzyc!DqA-$Bkx35o^TJ@=PszqBjKZ#-p zg8P_5cO7BV)BB0^WEGcg8wH}LThf>XHtdC%7Tc~Qg@~~XYlO5GJpzmr{w(>Mhg90S z*A(t@bpXBMCM1^@$2^O)yqQU3STddmt`ebThNsw73PISWk%HZA#uj=6VK@D-{*(V) zwXFWE_&pg{Ap_y@wq}*av8Jy1DG$C zPyL_wMbY}eA7}uAG$9cJ*nuNw>xG!8c`(#0az68o~;UreSh$~6$5_`C?^`1`&Vm;&Msi3yk2{;Ox!;Dqh;DngI* z!rPSS#N3i?gehAL5yJ%>Do46=!1#iI>_h{;Vf-<8FGTtDXw_*t`F)d;ZVGAvN`nBN zj*FTs&U1Gkz+R;91xgsSvIrRl9^|&W`5TeIpD;D((*o8%?vGy8c}>o>`kqW}9&xG( zckrzNi#D|agNxfHNm|bsYQXJ_GX@{3D~oK8I)OwnCdyrLkdBvk-*;ke`e2`z2n&8a zV*!=}tfib8#fjS&K`GLgf`UNGokRT)ujJ3EI%c*i6^hpuF*%9nbgx1?^Yv4vu|5ao z)nvA;9KV|sWbMgw>UE|1f|WNzN{m-XSYmb?cnetm`X`d9{icw>ir;CxD;4L&Zu3@meWs z=kl4!ks(Ai;;1*>%aRU8;MhAsM&fs;+tT&D=kH1F)&9qe1``MEnC?ed-vMb z8!9F5MDTxhptAm*z!HZj_$>Bv5380TlY)2qO)5b{I8r*H6we(}*^{z&&)Ona@s&;H zy+?PXv&%>J6Yptugy9qPK51<&H?1Wu?w%Bt(ae*34zc@s7o&jUX<7`d4agwiHton2 zAbajZ&TNQT`V4u@t8QI(zaUIzaOael&}u}Ngw5F_Y|6iJh_%Ph5GBZy&~pm$6Df2JO=UR0M+ld~_RY1DX}k*HUl{+iQDDeG#t zd%v2u5;v+-s($#@ZrwnVRd!j50#~PX&&fhn_Zp}iL%$Y!1h+qBTF2+gvr8dVVVMbd zat^z<3~-%jmxeU?5MV7>>?~zpbtZQPAMJaFxBC&xS+(TkmNy5{X;hKF<DkfBQx3461WmsdZ)9a{><0L+LDniokNjZ>@#$n=_W7%tx{0?XI~ zLKAYNNcLXzYvAx5931?X2c(OR@y)AQRvM`z)cfLhvPc_jjDQ^NN}fzpP3kc4=>2E) z=IN4fT`RG@>`xFjyP>~ORjkdU@GkSS*``~k)25eIg?^2?r#KsvIAg}mM-+N*W=tkv z-1(fddC;SUAuOna8GO&bh3==d^3U#L41!-A_IR&ctkt%L2f6qM@ka}q8ycdThvMTb zlvd@`AC#nVU34}Qd@`@EG;{Ty!I|l12iQQ2ZMwknwO>eOc6ELTa&Z)y3zAd6JI z#OH7?bc3<+>y*AvMYU;CBVkryqWrih1!e?!A#w_*k@B}P~ z;}@h!1PomF3}6HcsjcuT@U5|vi>v{H&88;AX)?6+Lvb8ScN%pFlg4i$>{||ifdXX4 zJQVeG-HIDyeTIj&co(!_z%vA3W;n4yC!j@5 z%ZOn<@J>_*Xe!AnF8A)`msE4fbL!>On)9P3Bys*;Xb#sDkPfc;2`JsHF8!~VkZ99(~gT&M=j4veoW6(X)w`rA-g-3T4@CX=hNWSm% zSiS=B^63vA89&uDqot$~4_Vyz4k_CuZ^2=h0tv=71|l&^n>2Z(0@0jG8pY$^W#@hA z4RiHrIW5r4lie+}Q-2Qn)6|nF5>%Y}=6DGDSNg33a*k#-n7|~sW^{sY!&v&Q(bCsY z6arHjumK_@v?~_0>8PXMEFz!$Z6*ti%UZv4Ob~DPb~3a&phvj^&(i8O3+4X?OIMY%p8se6 z0W+1L#h&g_No$59oSHAdr#C4^Nrc&o|Fqb{FK=9DO7HGz_3h-)mBxb>{ImMNs_ji7u)$c_fE%@d zo|5wWzQDVuDY-)lx&2gzlAY0Dnjo{&wWlmUGKNZHv{akUVxp}i$70RR19*+!|9MvQ zVfF0o2K%WEwN2xcmdD^((46Byilf=>@xpi<>pXFUzhOkmWKlf^u#0al zPPG3GRuv&u>O^#2(ud+oyUH(dKLJ-7dx>8C?|U}7cKoB2nDRPJE_u7EGnd-zUV2ek zFrow3re%eoiHP8G4ew*JFCInL%VY$5Ftqehs3^UAs#)}gQe8yw_V>GjEGE3=8mstu zu&4_*qBR|^bfa5DI?9eMWQYyd<)c#4P>y0+k^J1?e?C&gT zBp$eGDOc2J;;x$fRJ;6VSVOV*Sau7a)0FDjW`Q#|uK4o4iR8XLs3k^)eB5Psy7Sg# zT%RHHVgY(m*=EZ!XAqUKcj68}=LHE0Ou9FA0O%n(`UB9&}=67GEb-&f0?|I=*R_|rV zh~lw}R;_5sX*9CL3*#OrcsW0+XGt3o(A18*#5h}1L`9QfZPKA!ca;zwpYLZ7`#X^) zt96Y!t=z=e9Vzem?_Iwh7%MPA*26-tR<(Xq*;h-_JBP-?tx$G zS=0@~4;4GF3;a2DyPw@nCF_`_pRBSo+1Vo$AqsVJ^u;Y00OetR(ABsGH0q;AGr}H-1id1BxXav*-Z5d`m)VV~*@|+_crZ-Xm1CIu^VS;lS09#wOae$YdyhQ$b;b@& zPg|=!(Hy*m&gGhx?6T6-O3P%A%N)d}<&l{GgA8(x060X0zSiYgI3L(B!m!Pqa4u!U zNPKA0?}6xylmUL*j-F&#Q1He4q}HlwgJD4F6CF5La*R0k2tH;k?UEbRpb8UE4?0Pj;ODl>6F-0 z>6&a{d;SctrhwF-?&mrCJ4!UAOp@+}bLdo;Ll!t1$2Tq@%2B zGjosJYa+qpv2^iz{La#No{{@>^0$UJU(cC-7r1lRVF1YiLDS0fpPrsfuu2%YlpHs5 zbFilSdF(#()Sz#F4K+Eh-yz$o_wLfWRh?!xR#wk)pBblY4Y9d>+S8?eFZ1K}N8jd- zwQjq-z8JYYHkLcFFQH1%+0$e%S{9_n~5eKTXBvR75i*gBSnMPYt-w*0~Jhab3SBliCzxTpo6MM-hq&e zCLS|1dyVUf2JL+bnxw(yu_P2Pz`@+;=-r}y6^Z1qSAV5uOG=DqJN_k@yH@?uY<@Xvcu+=ihJ3i7$2sH1f|2 zn1)>2eivpDiWQU)0MYZ|&d@2_3P9^zZCwV@rC1(3h(Y*w?jB62z0L}EMeD8N0oRlA zjLFu%-9PXi#`cA$JpF4B5!v8eD9xn4pyT=jNF<;21TCjav)t^QUq2oSpDN|_Sayut zFl;_PLZLjlB9+`}j~NkLxSoEcx+>^cVHh?>)!nWd+u_yCIk!I^;CfYYdV#$y@9>xtX~wj3R2LR&|!MrI&F zUrgI@nx*T)RNv*-&WBM3XGVHax+nLKhg-{i(1-+!;Cq=S5fEWoa5e7f^`+=Rui`%v zG$5a9GD(dA^Sn;)YIeM!VyfkGV0eG!X6VG!;6bRS(^I{RGe`O>WE6vxyXkmpcJn6$ zOuyvGAk({tQ25cUS>IeN8av;dCA#7AAdBGFGlItHPY?4_F5hx+= z9>NxdQ!lg~(CDa%)Ll^{DwtvpLXB2^7vvGD?rKPE-%W6}uv+P%@YYpE6jFtdH@Gy2 zdZV|QJCE~PhaG@}FO}eIiL04;C-QzyH+vq_AMCw=cYh^P!p%V0qn53fDa6_251oJ20*i$2v#BjfUF_bJf2I9 zm%Z0-0V>qtX7PGb^9hz6(X}N4`RQCaSYma8=sABWmSB4uhftBNme1WeePw95E`F*f zWM9yb2o+eP2a_Rv&8r0A5sW}ghQy5TG;}3GS=pY`FA;)Nnm&Rrp7D^{>RLubt&FI_ zh6A8_Z*|%^fkaFm6xB#Dg@O1w4>#}j7z<@?ZCc7kwK@|bv>M}PP%ho%u@*-iMT|@> z&CPR@rWKQovx(xwf&uXFK6MKD)#{M_EGkZJZ_!V>(7q3uaB#BnYD#El#~p}u3fevv z#elO(ss*(NA$F4%UIJhK7UXZEzNiZOC4|dD(52i{+}ZcLO-fhevNg`bY#QARffOb+zk6uy(NqT;+t zKe=nBw*g*jPQm}Y_Q>DYPK`jrAq?`VyFVQf8%y^WPtiq-`pz3qY$R0I4lfEnunpMn zewlg`N$U}`*NAnsI{3ie%OHV78CQ~U44oyCnS|fQF~{rVP~o`nK^#miVy971khe{HXJqZ19Q+;uo#z~1G;4D2Tl zz*0w4oRb%JYJ_`%kQH);)O#J#y()188TB(2)uuLWX#7;KubuQ}mB^WbsgAkX5S{SRuKL8f+#Yh z5)TI-vUG29Oq@bQhLApL)0&^2ovm{0Jm-`!fY#=quj@Q09JUr)d=zRe%f1uoSy@H3 zyqrd&dvghIIorUF!Xj+k^73I62X-c3i|{1#bE-q<^DC#W?o&{Bt-~YL*6N3)x`QOr zxlQOCEK4r_*RrHNrFZA15|wxS8WPl=nX_Z$ytB$0TrFU3>{L(2egnJAj(zvFVH&f7 z=~MW<^(o}riVl0kWPu7l&Wvdy`sTmMH;3XbKsSAogpi^V&49qO2uWL_Z}c_vCO*-yD-q&)Qi))O!Ge zGe0Nl4JTHgl;c*E>?&PW2oNSPZaw#PjY4H zM`rAeg@rANavy;0W4$}_2+fzP+`N%OImr^S&hOl$W-qQ-w;FEN z7`@!adfZr*?vrAQJ%e&$gjOe^FA8`HLBFA_AvGrk&rkKS1tC)){BtLS zUAudxegiQ=ayH0Z$pR}Fq%tEue-ZQ8Ugt&VlM}(>uUbdAhC$95$T&jL5`@JM%ZSn6 zn|muxKY(18WT@^W;e`r+kC*=8VEZtWol)E%Tan3{jyEBPjinK ztdb)mQpZg3?A|GrcdKZ){ZVmj(m-Uh`D7<~&i7VLwm1Vfb~@2~J|FwCuYbP8jU1D4 z-VQ2k3V#l<`A4p`ZwY=YVws3hqUa^8;XJU1{fj4bMM!hP!a*s89+V2^RFZ|`JTbP_XbGb*ywlt|t4 zo&3?YwOy3uV_an9IKe8(k-4RaR7LN4WCG7AJd5NyZ*@^QQ2hYq3y&E}lbh{8AM{>; zL|WKx_~Y~?ZG(~Rvdbvsu#iVOm421kZBby)bT41u5Y0slx)h;0o!Rd4x8&PcKRM22I@b>v^vu3t6_*|7sJ!heh{i z0a$?wJjK0C+~&RHNY4)xt5(_`jmHISAiOO6;UV?+ly5tMnnNLl+sQN2Ru^AU^ick8 zNez;%BM2I2BwbRgb1ePZQ_#D5E5_GOLe%H06zSDj#A5TnX?A- zO}{(7et=%LB!&SJ5RLIYI(}y*K2LI5CD=$xkpi$LHt(K~{EfFrvWp}MS z0xfLhlHnHuOW_Ffw8vt;K~jORu+lk$j0HFaZ3Bbo%Q-0&Zx{F!zqT6c>7B({WF$`@ zY4Q^=Dq4#?{wX7|QEEMzwzpLsg7GSTU}>bA?U`8l&;#Ac=au`j|F0?ri{18vK=;B} zXk!F{-jQemhwReF_EBVRy|QDs!%&S~s948-T88te ziOr?=OZ>V#R`#B|j3jGZ-p~)5KbTZ+mw7VRdt~Sn|x;G`uGg6(r zGoKVbZ}N2g7DdN-xFmiPkn|ZjVX@kiTkaY5K)Rg5Yx}xEqawI+aOd1WavLz9F znn_kN?sVEI;9tG)RJE$pce494n`2ZnynHb_H7Icf#CWeJQ}iH+FFGK502&qkM_r;8 z9(%>5z?2v(!r!WKbn?uvk_Y0X1@{$%AW&oM9L20g()zW+lb9m_WzcCt)s;9$l75@egwAdLP7iB zxMf~fb^mp0PHh-zLLpj{X477SFJa8g5yGl<9yI_A{wHSrR{&3Ol8qj@3&0bUTv0Zzg97bdN!B;sBT8-*!Ntj;D_L06PBC`SIwlp-VZ{+-D& zeC8)HGpc8oh>Odm$Rl562e&KX&GqKwR&*Z#W~)VKlY<)qG&e?+Doj4ib=xKk6U+{K z%E_E0@Qw1$zO(V7T$o}(hxRacpDGDpq6PXJ?@?k$U;sNx>|r6RDoLBol9pia`Oeec z?=mH2xj}`?XZ5#xxmg<9Ci6y@+{rP}>8?L?fCe*bSq{xZY>wSQz>thIQgF3-ii@-% z<>!_{<`F_{d+G8g^WLG6z-*?S_l4PZY$Wj#9Ayu`zB84TuF`s(bD9R?MDyS6Xngj6 zc4xqqnp?TXYwwCQFw7^)rVIy&EiCKKblFj|un5LF&O?qaNki91L}0kH5K^3$p8oZZ zJU3VOa3Z2$Nzq(7Rs3evINpmmDePLmb8nIAV+p+$&96zb@9AXvt~m}gUG1_v<4IRP zcTEz^qN7x#IwC5iSZz{K$popE_nhOq3+Kzwon8eb!$o%{W9N{0&FN-2&)uOy=JO#) z=#>L1EEsOb15%>KL-dbJ7BUmX-;uTO#L06KbA{dDgRrY~o?i2`tF~IkxsKo`l4YOR zD;J;>aLr1|XF@NFrnm}!Mn$$3qLQC6qn1xzz0oz7^Vz5_mfTqI*z_uTxIF$fJ=PaeVRd6DE4S4NW!dQX zr(X{G8#N6hqmiWD7>TJ!^iRa9TAAfsPlL`!`ly(@!E0u{q&NAgokglt7YCb4dS?q?7;mW(^Di+)aLo-!#)n)O9hA8>cCoY?Yi% z(!(7Fv1JiEr-t3LCRmwp{UFo-F!-K@Sf(6DSPY!-bl;DdKzaY6+0@_fl7|7u3ry@+ za)e$iKOy~S`V@z70-xf}FE5SZ$5>_T)cSR|Pb|=qGZgfMlnj1*|3oQ6M)$kGw+q1% z_qS|HhD_b0G&)CVDIhSbE%a-1?v*1{j8q7n)w)Yp)csBSb3fk?AcY*7G*c>T(jn)1 zOv7<(BJcpY(xX`eL7y6D9D;WAs_@`>yIiYxdLf_x>L5A_~#LFkNU%E^KW z<;@?%`)d7mlPn(>2g)~F+Z8q$D;M9k%S+g}Nd3~z*4O#)(ma~N&1AeRdIgL+#bZ?{ z#JF-X?>0P8OO9ZoKacl0PJ?YqijZe|-61O{A4ujydB4b^?wH_iA$!?byGlN(yGL1l zsISvJfK}s?R>`6nW^>dpTe+=#?M_o&J!`vB_Z}j#UwKNV@@FHf^}DD@6zUPNltqcN zlM~IlfW1GjOzD*;DQuyU>(pl@zHSN0zFZrBx)2LUYOBrU(}?WChOO*6hKsXfdS2Y^ zirICXhi{{@sGV*tE_R8^`pG3551sx=db!Q|9rg89U##d7L5xtzErb(!?)kqf#Miq>nY@rbTcoia=~5l?eIXX-zATh>QdS^G z#se%@MPltjIwCLF56Y+x{<*F}P8LV|j3p3s9HllRK~IeM9JSYt0sx*PP3S;GtT}>_<8I5GeI@(xv?ML)Camw zv04x_PQT_=R|SDdz{(DJG(~tYcnahio4inYTKU2J8)a>-K=M1@R+>^n-GS{(MZnyl zv4gnvkEWNxH}k*Qhdnn4I3A@-dG%VIa2DohqFe^e0TG3(%8(qn0!P)z=Bsz+8(j(r zHt*!)COq1!qptJy<<=#BKB7&oMt)L;>&r6|3VpMi$<~@tWn5yU`F*`K*Zr%#b>uhe zC-)0he5vo})^Eogjcx6#n)E>zpJC*;+on(Sv)fi&?H{oSU(}H^Cq(kPK`q=k`R3c| zCqf-^OikMTD9vkLvR;Xg7AhYIFUZw-DrKr7^fF!!jYA!m-YqkYjMJr@rQuc2dR?ia z=O;~eN)`BWKQXaj3&c4*BVMgAd~rqloAEV)Pnhn_Xm+7)T6^9rjxER4i@=*dSdLS! z?{%-DXT#RRu;7We3sraHn7=#b0o`ztU3Opxhgy_=?rc9Qb#NQh1&v7EXTnieQ-B|U zSHD&&li!$la!!YBqUVc8X9DQ86A!a8RkdhWz7iY&@ zt=v`W{4HM1U?4q)PAIsA5t7pFIeDs<`Vu=jhA~UGsKv8%<>|L=-lLxTx{@RN>m|{_ zX^l&at)ie_v=_Lne3~-;LhZMoCwZc&J_blb4^3~I5E9IDi*<@MfzSW#amwt7M=G2F zkBcX>N`@KiCK?X+0GxuGcrN(17@Yp;ZuBa8$63stI{@F>v4kqE3`!5{}d$C+6$ge8ZgXay_I+C0VxeAl&sb#5J3!moS zvdlcbBqjwYO5`fLKQa`q2Zi%=92h;t7>xAp6 zk0LP-L`nUd!#~~r1vDToe>Eqm$5*A8PU@4|^SlJpjdrGnHR?_PcW1GyFW>7o7hZKzK~j+w@J#&%E~hKDn6Mjb>h6aYcPkno?qy)5`Xk( z8MG2bG@nb2NU+FWQ84GBi7ASAp%exFWkRFHIq!@a%94{m+o;<$h`IG@YM0I3HIBRD z9N;eB9M&tI=`rr-zyzo;<^?>NtUeq~XzaH7WOw#j79YQNVDW{~!eomTehV^fTMw4g zm8Q=Z&Ha$>k(wv_qz2cVltRIyg2j~&;~6FqHz^HYIB90INGR_$zG0R7C|z5?+88H? zv-R}e!f#m(+cWJ)Y47VWa{5pgO;{Q2MDBdw@dM>Ty>*4|)^F0i>b^>kM-IBLL-{_5+iw|;!(Ckr@pIfwv6w;y7!Kb*hG@p<*sc^Z~! z|6~Q;3mtcwx0^d%DDR zu`*_&tf4XRv0JCM?r!;fPRdl{r1qe0mW5lPW5dp$Nnxd<#e!!WSBwa=SY?`zFS|{> z-Zg*Xr@5{{=Sk>0{>hy;_n4Q{igqDJ_SgrnUjE4iuXU5jfI8)>UeZBD%9F&0Ym$l? zJ5=hglM*l8;~6e7tgme-=YQS0~n zRk->KMJ&()i_~IbVm-%cugMXmQ70I66zH)qO0A0vuS{2F2iN;mNFDd3PLNIuL$$;O z&q?pRH>{tmoKF16@SMaiNx@xBMO6lIgShoNoMpiW^Y5?!WJIO)8_Ty_x5D>?U&kr1>K!*C(M?Y!owZMR9St@DXFFSoCBz^t7W(wMmACx zB8B$?B9FZxuFCXs8+qdWto8T{mf-{vPIPjK+5QGBSsX1f zP`1{Ku}=8iCRTSTOt*GI&Av*3SUqwL;X>Z0=5_gVkA0WzDXPD#>(`BH(EmuQ?H;Z_ zF(E?tw*9b#&z-(n;S+!BRJ7SShVxED0?H>ubR0tkPx9QqNI2z%6qB)KekXIg)>{)P zLoJl*MOU;>{eHS4w{+@VmoYB|jbnMJ@FN*R1*#wq*=jbgSBh|PuKRn*LQTkZnLSZL zH(cLZLt_91RRsAJmbBpQhgAEWMR_m!ven7 zj{B83Mqsuf4Ah9M4Aty;Zd`HkLEjnr70?p@UymvOQ$3G|%Nm<$Qb$>5$l6d@=a7Jn z75c0CQKy}J{(iDPeucYiHj79K;o=!2)S5q%gfR;){p)%lhB_ij3P(K#4V zc&(N@<_t~#yV&L62MDQQd8z5jJ+!??o+Ij{m%f4 zrR|y$e^BoOVK)ZAxLO*XsiV}3I82w2&%1L9TFyDl@bG$YJ1eF>HrR^uXe>i7 z&%eJ!*6SC&wft@H#iA^Hyn-a+UesTG0m zIKNrLF<9}$uF^RYm}CN$78dq+IM`hFjX(F13K$*j6tzT|_(?y@Au)m6dvHr7>u%mbB3K=qc_z zWIAe`LASAvw`*n{P<`QY952;4>bKMU%ZGYmsdLG8r6lquo6nTpi`_|_d}8V@Hj%LM zS^9?bW<>+s-!9wmY-&(T)x^~?{O2vtv%_1~BAH=!&baj}Bu2GS{ErkXdqaSUH?rmb zDPH6r`VFE`zj@)->&GdU_G8V#PY5G_Z@W>(K4~UVhlit##V?zeTNu)PqQ1PgYyNfd z8Mzuf`*x9sIDMv!_SZJB{xRRzqRhvQew+=A$>$YY!naRlw#h{=er1T&R*_Wt?<>x6 z28N)Y0C={W?cwfjbq0Hl@1~@rOaZFsf@k&G1YmP4Ynv#~0ak0BNSn5A7Tw<0r&Y2( z4Y%V*|Cb$S>P0_c2Y>wd6V#k{m2>HPc?NQSP)wd)4E`L*B;Mys!Kyg+uYzDnA9g`1U$>lX6&A+(+X`vLf>bj@WsnFYC%IgHP{BbO=ZR|SIlA^>DAy; z>SbkD!&&~`#;&j4ZGxLZ?!cR+m@XdU5`EnL3xe97wi>$@bCKYrAB72(>K+_-{vc_i z{RtSP%RblyM8}dcEMUeFb#5(v21UAv{X*N?Is z(XU?ZQkuudpWM8EAzQ=b1Rt(FEA+JepYMF{<@JwlUD^eynEh?S5=a}$3&H1C)c!V_ zqxEYDyFB=k+OQ6Z9C5v0b;&~2T=d=#q0=SOKZkeLzA(DoKnVp%osU+O#us^yP*qP@ zd1ApQ>5n;JceanOS^OfJp-{#5DNE^ z@RdDQq$cKPL$j$@wq8m0`*SaKj7^`_gKTC3!_vYgVFp`%$v%17>l(}vwY1lJ$p4w1 zeb6$Sjo-Khq&BLyD+^BB1I47=^Sb}cvt-&+zaK-37_dP@Y7$Uf6K(ZemXz54D&>q1 zw1{HT<9Z|8T2A2nuo2xX)6w4Eo|c%HxG34U6>4(9n|h-Eg5&s!L$08t*Ue5k6v6l4 z#i*d#TWSjDj=>2og{{DKO;gmYG`K6VZh5WjN6lEKkn5P4Ov@7sZ;xQG$$D_+^^sWO@n2CoX*@KX#vGM5xwS`^Yn*=KE)wTM$R) zb~_r5;Pv(OXAeYaGPU0udAa2bxMMa3tkpb;U)?PQls$TcpcXj$WJ#LWo6=zdcsEbk2P}N3 zYXTvBG!}$)BGMGq&eyGVLT){;%ck_X*6hG>Pn4+IxnaPal#2PvH@Zi9lviBl1}A*x z5d!(~bS6qp<8z1B6lX^Otl*#>W?HT3e?ZtaKAh#V6Mx7)A@VkAPJWZ)x2)MMj&moE zuaXX%mZp>;&IMBWXT|h4UGtCA_^5uHHW$pL?ww3L1uYqlVMzzM{zfvJnEU;9(={e=Jfvu+hf=*PBfYZLZ?v#IlEq~8 z%=~h8PtgkLh0rw^Z1mk8tu55R7c}qnUHAocuM7YQQ<+=rMl@H1h=gdUXOBgVM*c>Q zLKZg$shCX;Q=5-aD~0(ZkAKk3xqQCSxc=M7d6lE^%kYjm{!;?k;f|_iuV=SHM&m1F zvG;4`^k|jXOp685SDqiI!M{KJDg$5{!h&Gefol8Rs%r zo;rQset#*_w5~`&`hI${1qe$t3)W8#UnpKVYM0c_BQDN!WdrfgY&W|cA~@=|z`)fC z`IF6CwR?3mS9m#`V9#e;ak|fKJeAvSOJ@sT_QNk}`-UO+yr4+){DT;=N8kB5zV#8e zJQcC6-ZI^g&b6^vjb+LnOAT=3KGu_{*!czLl1|P3$g1B&d|E6#W^rp7r|h6$|5lnH z(USJvafcqeMBmQi@(~`#Q91GVLN9AV>wpFjjnaOVh|^7yt9a@Blb*-$PYajQY$<5E zK4L*iKVeBvrbZV{%$VOXGBCC6?`(0e%;6E^T?QEBeKCbF?Z-<_Z+E!th!)q}R=PmI zR|DOqIewx5My%@?DQ<4=tFM>oh+~sHu2?q5kL(sZ_Sx)+z0dX(J`T`t*XCp}A**1n zhGlNI&%my1M2Z*mza5S=TH?MnzG~8SCHH+J5ZCaE!1+4D{$9F6!zbJJMvoMo(vx>TD^u z29&WWg*VE_H1WhzQ=mD8=TAI(*z6PJ$r$2ebUc1cf~A5~!D1uNp+0ZU=(z|Cp}#H^ zd~JDeRNcr@?_BnNU-b9Ujn)Rjx*`{QtNK0ZKb8LW8G1lt?!Wpi%<}C`gwQqLR`*5`xr>grtDv07DHuFfiwN(QohHb9;@ zx-FVXH96pTYR7M%wi8$>0qH;(47bC;(jPc(7Ff`@H>w4b^?Cu2t*4Vt`RzHZpQHED zP;V(Gh^25Mem~3*n~59AD2zn1-C-VjxZ%__)VOnfC_x%jHZ~qrf5@Yv+EaSFtgx`> z>iBb`8rNKkF_5+Ze==z%0GaGzD~${dv%qe*8Y`yEUY2iE<>+ysyociQm#{&(W8D2G zW!hz0NemGW-6cWwpC}t{h|D3~|lgCTv&hE~jk&nhCKySv33DZ{)bn@X2$glF`D2vjh@DU%b z^i92y9Yc7nwt>d!#q2km`BZLx^^&ood>djCi3QFgJfmecF=LM?S0m%IWr&Y~2NS?p zGj83k8TPNRcSQx0(;H9p3oGNX*rRz6WLE`^S?;#G26dfletGP{70Q#4`3r91;}DE( zg}Il0l&*U}$E3xt{%s|T!?lkO$FvZS-LGFeEBWL-vaEOeUY(>^EMIt03`126ZL~?0 zpVY?5p>3C5%ZH)|u?zbzMA-T>v$ch(aukB=hXXb=Q>gl$`D~+C_MP3#XUEjM_IX_{ zOK}oeo*6Ok57CjFtxL=b9In_6tfBovt#FtxtxCpS>T~cTFWt)t_Am(-j(&x{{i;9_ zu+GQjrXC^tobD@#4^RFq*Da)s9sONydg%GlpFY?@Kd3VQ8)IeON-Ec)akRsR^=3$p zKj-Vla;$;j;ht^kreXjhH{D&oPC*E1FeyAZh=Fq=;mX4D-Cf~|vt z{!XqHubhl8^5IKhoUc%qxEZ3OFnHEpZi=yGB67b66;olSJ+|H@wrH|8gWPSo5k?b# zQvys4fZdWdx&2iD^^~wbgk0SbZ+`P89%P|;oHrT{_By;-{U*VV_h}%$%TzXVX366% zIrfByKfKyuV`Te1Jr(n{)*3L*)zxz-2FSC>x_^`-Z9H6Ysv|rQ08NlETbHd4s%no? z?ZX4on{d5QK+bbBS4I#BSfmH~#366biHph?UNA!|1IBSMN%j)ue>6moU3`FcEmLsm z$%d^p2ir>CaGF?#TR-KmXiDzDpjEFhTKo12ghIKXc-X7^roh{jVCvM@NWFoQl(bDH z;jLh9R`YdWcS7BNktF_QMz zZ~7*gbnH?NRX8zLwlIIpyPNK6ufcCXTS~hQUhZWpZUh{*@tqeJsDQ)vfkj9fXQ}g) zGKM%SsOSQN;!B6*@ zpP@Y;SKCY-VhraVI{~Sc2ocyKI7?Vl=06A){7KcBQA=%yE%+y&61-iK0>B>2^@u;w zc$(1oRd(Xr;LL}@4)I2-i*o4SaEcam1p1wVreBn}$ifMG`dzR&mp!uO6%A+OY(Ejv(tXog-m3Wv1j!at!(gs^mw185cms<)TR{`_ z#+3b_VUt+Bh?F)Zduv7CF z(G|mB3Z%;EN9W3W3OAcf2VdU3s6D7WD5XW`KKny_CT|hA0O2MYl>)>u=WcLaCpS`S zIV~d=mjQ2?qpjVbtmad&_Dyhtt%i`t8_P*zlCAUxRLh{86}%4~5C}P(ZuJ6Ai|^aI z8qgKXz@{Y!T&iPw!V;eXuf`&!I%~=G5K_5WyQ)PIOLv`P=}dcWu($54e!a5 zfuQZFku@)SH#0=aD8DmoiIRHkGj|cQS+M7`VKOlpp0f|IDdoNOyJeLZl}${2e!n>T zi|#Bye9YMTQo%-YgdOCQf92ya3V7kg?jTChf(e}1wR1jCkFa%eJ?{u8EvT($gFY4S zL`gn;9pwfL&(E77A1~pwf1IQ+_!#y~EF{DI_oI{|&YO%?cX%1IXg?;}c$K_l*3v#@ z&Y?lZ{ptboAO=Om{tR3-uH6}bxhSSRPS@7>l$*YKyTdH-kg zn5n++vZVBHb{byKH_PXIOMG(f;)74Y=B|B$Ldc|Io?k_8W~C?Fp>AS#^NmNnPM?c` zolbL1i%yV^)-z@4S8-0#azjMD@|ig8-=jElW;`FeCBAc}*?rW1XYJ4&&Br{(T^R${ z6`7An7VUbFaU>r9VYosi;%8zxVQ-f=s=%_-r?QKGF?o)dv$rCRT*c=kH(qbXKB+0U z*Js-!Inx#Q5q1oroqDu4VB~Lxe|bM3=y2DUZfDi;>C*@m-|4DM_=8mjYnD@Mx2K!& zYX>=V3oOUV)51mNo3X#8>m0=&Im)$&w#d6UUyWwhSo08f1~~9J7vQBKgY4rK zVOr_9bWu%E2T=CUUj%DR+`k+9aS2!^1am-bG;1cYA{B=R&|)uuU1qKS-7+O#1;qqd zh1CBOD3+DZYy=k-b7jF^0?*HxM^~(G#P4-KVa)zD1!RLB0UTl>ox*aB=7vT_emNa# z-j8AnMxJs8?3vg9Oud~#+2Sr1Ty`V*WOxXxnZQ-zW|U2PDt_elOrlRLMbE-$mB4Ps zh~MwjD?KTfL2r$zH3*(3jqvGf+Iq$qMTOl?x&rXS-eEStdB)c#UOA~%elmd)PG1!|8F0JwhLYcFud9RSOFkf4 z=&~x_Kr}fFYvVs>3NAU|dj~_;>m>ny-84zbuzTbh)6aPT6Sy=QEdD7hG}hOy zth5&!cc0ds*!B#j;i2YaCkCd8(kCcWIs%MLo@g%!ht(WpW=(n?#;p5v{lutw`P-BS z>xh3;I&lJc>u7z-N^xqe7;qDB__lj?k{fWg+ytKbbANtFsMrPG&@Q_sL9cZ?&9qzT zB!VK=y!w7$*}gL)+J);fJ-Rlr)B{d}894UgOOO{OGbt6BdP~M?aVGK|StbN?jlK1_ z-<~ucqyxjZ;(flLc-On+J3^Y@1g;g9b!jI~fUXZr&ZIMC>5NOfT6-n%9$-UfJPceX z0Iq|KOQ_I`%kawgTiyg0Gve?uS0e?0mJRg%o&qu{YE390brIkj&W$lt9NPAZ+yNU! z+f)8FK~#2r68fkTPv*6`9M5dU=e>%eRKfRE`{h?_u|<*S&mth2(zBjlbq>toO5nZ% z*9e1quMk3Tn#M#lc<4DL20Z^)pkO>QaIHPfZAOw)?Gbt~0_wcHn9n z&GySeBoZXiV8VVheKnL14c9?D)wF1@TWe$m8*0XlXRX!LE((C^56-PG$uSg+BJKhF zAikq*F1p1RdC0 z%MWbrG>zh3UwguksWYnKFC?Yqr(u`^n0FVPZdp-vR`}@yy-}W^@-vR_F+aIwy=cYas%K-h($K;&IUC(Az&38?DnLN=>$Wwvde$1I8fk-V z`9SmEJ7mJ;hDeM@k*AnL7EZ%6FPGaB1L^w@?H?2QE5!P)nX0hoHm; z1+-R>rtyu<@+2+4#7A-RJVi9Q0Qr8ot~=J(N0%<|bM5>hYm)smZvrjj&t-WUWl9)K z7Pp2{v~RUZ7Y$}&dMM^t$Br>^k}nw$O7^&{fpg$$aOS5VrBqkcXSV#vFAaKlhyoshF)5wU?ytP zvQA{O%IDi!{T^;*q)c$)3TI=B&)toP#-QzyyFs$?e4kTG?2%``4Fg(?&;wGEU#?wO zU2L5QlT4iwaJ}axIW{d~x`UZkLct5HVKIj&Je;@t5F3c_T*EpODY$)SkX;0&yXe*g zHao?UI(bpM72b)$n|_S_Tbf00zVl9>)1|{1z9V5M{c&Q1#^T3&n<}F8R_dfDs@B02 z2sjB9y(v#?&A5F?r2*>osL1h8VPVdXA^i|%NISz$`~ioW6$z3Xl0L#g0FUuG(yIm51*+Pxh7ldK_vp7yTZ$2z|*d;e7TQ_sXwM3GpN})I)pvARf zD{RWMK3-=5jP%pIawhow_Aw6^Hv5Cv^{!Bi>{Z6(M zeLB}!-4bO5PlIEzWcOSE$3~^n@?nQuf61=<{ezC_ZeOo4@guqZW({P6P``1;F3;NU z9Je^*niWbU-<{$~&6Op`tUC!`kOrYOl3o7tv zZ*n?2SVMhn$#HK_>Pr)wvSweqeERvucQ0m)r%zC5$tH?nCB@r`?)Ue2Zj&AKqh@Le zoR87XE}DBoTJ&FqqjOMvsNGwXY4JfG&ictr4AB%{)^l95yTgU*)usDhyGOrCxZ9dk zLK5y;V%hioHpOl*N?7r+kV@KQ?-@3AS%S3lfap&0&5H@VC5i=M^BQL{dyDJ1kjooy zLf8}EhFaX)F>l~Us_FTYeVJ*#Cal)iz_I}5qnw|NMez?m>~sh3D#$Ck2b>k>QwW@| zrnW;W9BFOvXT5C zc=_I!>)Od}s0-&^WU|kH+dpxeXJtHf)|T^akm(IO{5CFzF{_VqO6SoWO9{v}YQ>Yt zvno@{ZY}F2I!YvlWf>L0q*AUG<&9j?9x3pj*O2sY{=V>MajlxYUW_EFlpjPzF9g=( zT(*>+buPugc9U%2s*6sP8if(*ervEVB!uf&I_6v^&Kw|f_$vOnXfQzl{VC8cQo!J1DmzJ(ti zQTn6K5RpOoa4PqWIs2VK6R_opg3#Qs2(@pVh>}3!(Ze^D9KCWCo?uR6E1=k)93y_C zDw0HDujuSke9&1AU{E^sas5-({`22bHYm2Lbc|H>g_8rVKY^&FdixWtir4ZmE%1Jp z1GCKz$Mcf$<^u&q7w3vl6ZLs z5G5pO5FCI9oC9M)-!=c(rUIqvwn9 zA*{JkdmTn6UTp*^`^FwJ&1h^C%`tKRD+{-_Dv)3ix$MK?7akH!j| z8elt3aa>tiJEVlOM%&~R zSVxlUPKxy(1UCBhDWEAmPte28j%s+pPSM`PFR;Sk-fUO)SpmmnAu3t6OCrVOA!F*H z%n$2RdL*)nlc0gwLlVrgirLDV4nxK_f#*T>`KcGdfFjY7dGg9<>V(|N!>WT{m4GYp zv7NaITFOqQ*v;Gc-%dVxZpZPZun_kShgm2T13h7XlYO!kSaqsqJ&*WW?YUxTVki6-8AJV+_D7L^$!?Lh5i2%Ui3plYYA!kITGvK}KR0*fBDi45gN@jMohn zN}bEW5jN3lyIwMLzwApYkIj)zVO$4t*o&HM%x-l$#Z*b9jZ@##? z#0op%W^MmW;X=1f>Elc0n;!zjYXjQsHn{*l;N%qjgDUy_n>Hl&egT%v$s~Gvl6=nx z1$`e;yrFC!1C_+;<%0n#_pA(Ak1I!_iX?Un66yJ11;tj9RR!)=kcSZSy3mz&C*leb z(|BuH?(`l*vTHybYp~B;m6N+U-OV!t*(aXz@K2_<$8g4|2$AS9B_$g?dw%!Q^9V;^ zm_q}+T=lVm5udsLtt(WB)JKQgiq$JUUIYX8K?It?I%Jf1gZHgLW&z`+{wn1GYWWs- z$>3%TR8{B|eL`7JlE2#ypZE!ONUBypN03=40@lQM}dwx`6+&9@xJtQhvZxJqle+&#D1B~z(F8F8_S=!I7k%feObQGYM_`U^@ z%^?EM=)}<*in13;!r2o3(;AEdtv4+}di7mf%d7XgKR?T07w;dnEkJY?;!u0c~rU) zZBy*FkRpDTUhp}C5$i;B;XnoJN$VfbXcgX1ck@=o>l1)Q!MHqf5nPj06(oM;T}u36 z9!eG_ z^_wT%ZamUHoE)#D$*L`B6ig@o7s9TOECBMR`FRqWmlf5CGf^|W>Vik(A>s%7tn?`1 zr}u9oqp~vY`Cq88hBzkx9=y<#CL=Lg<7UHVXaqajn^^$%Ex5ouW?6Sz_og$9twcc7 zw_8ZR!or*^e-ZYX8hmGE=-;q-&5e72SbDzRv7Z4wo4&5QiZZx3OcZk_*ewai-Dggd z_vw@21|%M6+P4J*Yr`+dCLP%Pr~*2d2hXQJ`~gs*jxPduMLDJ6%U)-WXZ>EMK;l_J zIFIk`y>4~TbGa!a9K}c?=_}g1*EUjHwI8|>0;gNU7XYIx3h*t{F1_pwF8=$015hUm z{qN2bZhd5#1#98!NKhOPD>^j{uVR^S`(%Bqj9}sG~>ohna=>mB#B1(Z41a(0DZz( z7SW*GR=4~~2VGT<_77);%Si~#2vc<&L*ZsBEz62*Fd_Pe zck0Wkn@2vFb2K%V7Ns&q3e`Ju2vb+z3zm#n36q~=I?k9i(glgDN*JTZ&DXAD=o9ki za_Ov&e+u|2(~^HL)cu=#zZ#0O#b!2xERk-_7>N4l@R!BFCkxPlnd}gnpb)U*VALL% zfQ-Ar67PUh@j^XR-lI4aYMswJ0(~arj_!wWL)a}F)qA_J(bYJ}h0R0(+Q9a?O>QdB z$4MN3B{v4bKL7=ZDT-O5?NDd`d*l@6%?v^QH5$a}#`BN9;ZPKTA5r%wpKjMqlm2pV zq8bR-YRiL_mjKmhb?1Se9h8yFz6q#0ZQOG%An~!zd%}B$YFDu_ml}iV5goGdQ2+Pc zz(dUd_ygG=enZw;k)J5UofU*3_Ri|``<^4 z#@|xIh7q^(Jh-=C>@T&wh$f3eX%FtY`^9#bmA%)gu5t>{)UqQ{oU4QQg31LM{Xo|3 zyB~GjP5Zu-o;+O-sEBGHpWK?Q5726wxtj-titFi2pkRg!N1Ovt5Z>KW;WBE3`_yB$ zmhad)ImL)H?pai=v~(-AN(4(3vBzFKni3?A=2^!fX1+@0c?I= z4(n4|`^I8Qx;54a_)R~u?1083yG|KIh33#qU>_zElh_3KV19GG8#kHRz#i?qf(cC1 zD)MW@kK3|fW-(@i8DLm^mZE90aNzuxyw+cAI8PWy!EfL`v}i~&n}AR4;4N9gS_b!iqH%i8C<^gEczw!jn#)BUO}U$)Jar=2`acnxfA z$M6E@23fRm4D^cJ7lf!J0uSa>on(2k_jD(tbb_ z=Pl$~CfTGt&VtBc@ZA3SNKTQ@xSjq@{N&}}F8O8`@jh^g|C}v|PA*7!5=;|x{(ma9 zxP!k>zmh;9s(f96?>qS1Y32{qDmL>52ay1T{$@0%yM%XY*iP8&qCM2sZd-P`Z!jIJ zUt;!RlQ#KqTM2l%uE1NnCAs4aGPR^x!W@AgV?^xfQ=qjO-Vl0!&<&B-{dp-TN|wh$s~3t7i%~luER(s7ylpdswZX z^~dC;?VjH-XnjST*lolR_<%rcR(l}DZ}yG4Yq7__buo+h=f2E7uRuOa=m6xT=^ro7 zS`z_Hd1~W@lYQ>>Yh;;qyB-&;j~9D@far}+%mNJV>!9h*R0WOIGk7`Zml(9a?<*TF zKq>Hh-`m=x9vF!L@L5|}Win)_W-635jR=MW+05OHo=fR4|5_j(`*YzXIRInuG?qYE zwo$~I8(!u^i+>fb7T#dvRpo7QD|cV)vju108QlnvY%+>7?9tn2G7D^8-_?i|)nwba zRr14y0EGL$#B&k1&C~3%E-dcE_(5qb3JO!kU82#~G-r+u)X&}$mZ6Es)26+^-Lk0l07AGWvan%x6No94W!XnLV=`U!`k!Ybm!Ucw3x(Z zU@4G}mW!#3OGfgeZX-%zfM65@F)5B~_m&j61g*d!Xj~Qs&Ssr-=C>Fo+3zx>i2`b~ zGHM95kZAGKDjB?L8)IYvX8wZBG+u;}3(&$k{qb`&(e5}cWeMEsFP6B#@p}AxD)@!c zM<7zgRhik2E8ULWhxEYToC?t{iz)SnRJ?^5&P8*K&)QVz0EP2e{Y4q^JA5;d!M}`< zmSdOl9Pc9ueixPf4Z*o42^`fQH~X8S8ny?GW(qIW0{ZlHVH;?={-Of6{%8aMwBGwD zHB9z+ZE>sFhW({o_wA(i3t1Ur{goh~(KFyNVVfUfx_jpfKKP*R0$Y3Ge2 z06pY@)R!oS89_z7CSX9zx9^n&RXUEdIBkPxNm=p=u>ZjFGYKdO8V0J!CWj>cBu@}< zH^ih-vu6QnJuXnU+}Mb4{#lUf&Y+We zqp;xo&m1@ZxWHvVB#{5H2LK_)U^HOk-a;N8a{fo=TcAS&!#f;+WWRoCRYcqGXt`Bq z7|DHG>b*6a(M^O;uw@i?>%{IP@o0BkE^X2-@B|f!%W_heDDv0JjP$fcPnRV37sdDN zmOPh8+%@v&qFl}1y`Dtj%ozlO5Z*T89|r=RZ~-%c6UD7@t;fhqXGiA;mU*WzpqYXf zz4)$mw0^iI_3w#Up>)dj;&;p2m(k+zSw>#yqMUPo+Z%5CL$eZ`R)7l2yR^LQ9z+Q? zmg6wl@1LpCy^V){ffI^iNNV8(u-k9|bPG`BIh?jH3`&cBDtiV<130~1aZNt*Y{+-6 z>aH7974uUy;{?7LHHV$I4?NK&g~a!GVwuYDNC9OWO-tM@ zv-oxnvsu)>)CIZ_-yS~y>2Sd`w3EDx%deHH$<=0W>H(m1-_V#k2q`z5xD*E1au+t( zoBgoh&2(G4G4UzQ(n6T+e5^M0i^VW;vrwt+hJBC)Be z+nL{oHM@RJk%&jn6gnzzR{i{(YHS1fhX;=57*LaSxUGUWn+5Rx5zL7Hw;baSfrcil z??*=7)yK~oP{0yPeGN%$JUCzC=1k@sj6>Zu?0e}&N@BOn7EOKDfL01qY+O@HBkJO+ zFhSzE3i8>eC2FU3GuQd1e&*?O$YdTK9po_8Q=5WVp=K}I=z8v9o@k3vsjGKK4WPB{ zT(gWPY%%AVcRQUrU46_@fx!c5!4)IQ0{HEY#3m1#!WO^jD)xx@{NrNB#PlPv`FXwT z{=*XEYrb3H{ppO83fRBc)e}TR@)C9Oyn*fWXJq0Fm$pYhIDys_+po@ep&5m5c1})f zE+WLd-qFz%pJm|eNpUcIm-kg(Vd2#YHx@-5h@(k4*({))^t&0pvo>iu95W*e3rGVm zxI#1)ZnG2s#nRHH z5u8K;e2wNL8o>v?ep_uf14kFV2pX#PDyj`)OxSF26?Cb#?p}=Q{>zp8$AA4`d?B&p zC5%IE2WSgiY$v}Qy|Wn7c?n}dDt7iI&fRW`QKZ1KUJ;~LraTP|_~KmacfK|Md}PY* z^ivRo!6T|_&DG4lt57Qd0r=z8PPiGVQgwz+^?6(AvQ5y?g;&+~)Zv56Zh=M|Y>ADD zaTOI}Lv70sB`%(QMjx77le68qtl!u3U0nMR?;@sk#?9-M^5#%>aAR5*F*xd(PP*87o7#+7d(!uRUMDHY;*Eqrm}K6qF<~IeYPIqrvSqi>QI%4Ud$Iad4ne1JD-)stg*xz>r|>1kA^IvozmN+;xUm)zl(|CG?>WT#tx(4f9Mas zA|3SyHMtlqp0LdN0iggcu;KDHWD~qi2UD?Kwocw$XKmpQ@Huu0kiByqYn?QI zzxx8C+)(!>%dt#(i6R&+=y@oxzgoKs78>K`|7))LJBtM~33D;@30^qAFIDLC78`r8HgA`*f;iDkYiQ2hG10z>LRC z9q>qE1#OyYW*LWAipv;^#$`d+IH<&+kdP%!b^iqJ?;;bO0&6t^_`PgFEU+e^LK9_E zkn@baadpJY`wOi|byEk9rC^=Gy|?TC5$RDn78K8X%FA#KfDXp=Js-egXyXQ>WH3N! zgBQ+n_$`BmJHoMgDioXqV59;}jSLf}l%|T2tkCdeKq0H$YzjsLNt+BO!5}sutx@9~ z08zb8jRFA}n2{nGu7dSC{{(+TILyhVn)jQ7yY&$)%KM z?vQZ5wvRPU>zSs!lbz<$2F@doBw3HoMsSXHZT-Lmk>))pb(1)h>WbQ4b21#+Voa(T z2ZkJ5fcMix{%1rAMz=kIRDQ>n5YWBzVZQ$KEp}6*bopDsz5SP^q;YPqwsVuT40p&0 zmiiHwqjIE$H187({AG_^3>J4jd2$4Qya)jcB&R0dVsB1Cyfr-?P_*BVj9FMa)9wI9 z_YCoFWb*N(V(dK?f9DTb-xwS69zOZt;?9zJRsaZj>8`G2T$+z3%?=gP}*9 z@7HJgER$ftwJdH_#^f(k^*9OUl0y$mRriMhqQSqf-$KXrA>$bW1QK&Dq`o(`2PS?l z8csA~v^&f2o5EHGe5i^e_Eg#>9RYnX=2P2SsohUI76?L^BWCwZk(Oc%^~+1cIcGWd zJKp?eQ2^0Aee=ctnp~O7@Xq6Jo9tU$&%WPoEiW-Y}o0Yw|=MB?&bQ?l*YKfHEQtm8|i-SE1QT2dPyccrLD z84}kcnP_<9zrSVsdEusAv!H+AmVNHFhd%MHX+eB$6Xos6KGveN#;=k|Z2lgBHvUpbyqXhG(@~1zqRB_}qQL1;pV5u+p;+6?KAQGtPZ)hR@0ea7;ZB}efm0@B6+`jb892~vNi%+V zWw|0hSikj+tMwZQJN`!G54vQ7nE{5MYb@MH6|`-HSz;2Zaw;^+etXLW;-A;(wnwx2}!ILA} zVF}?xn)?vZ0tqL;h}AgIbHpPY1GE`}6LeXXf5GoR(`=Tp{#sn4@V@CnSI3f+y2fEE}g)vtzqi#3J=4@S`H`ur&)MJMk;-OX^paTme#7PpW=tqxz za>Kad8CAbR(*1wPw%&8=hSS~wqTpp(9owTD9T|-}wuak~Gi+5Y&PH)xJ40S(LiSFq z{u0_y(y?YPq5E3rgMimoFll2Ag`5F5PH;9Ed+<24*@+XjmdC0@Czdp!d-a!4C?I+9 zs%qg)!fe%?4#+T=`A@yfk|dCmA>JGOh*9CBCt38*w|_=II67*$pk529i`I-7EK_Hc zrA1%SmuNGD#Bd<8qh2m9*z~P($+!ycBgzh%X|Bi9>I=p|N?Ms`t9_Zqtsy0Zq{F6RM`S;K-q6?%I!@%llpsY-lui{0A?Qovc< zQ2Kl(akDSct;v)kVN(cYNdCaYp!i%1c<%N=0sW{UZl%c`SE2lM~0+_4TF` z=uiN@Dbz)UjX!v&G$+LXmBf512`}-J&$v$WexP<2xSOU=>gM8D&E9Wn6KmYH_^#30+ee$(iBe2reHo}{$`I$xF4Xjn734;@805mLHLaBNW zcPH0K+sJg+-~?QIQRDRRj=70Xvv`oe-&Z**0%~?>vdE4`X`@OeAdUm5jhj_bV%h(I z(Gx5#WDt#4>gk@Ig-y~H+_`#aL8`rdE4T=U(mVgC|Wb;@Ed5VpLqBoytC39{wu z)`7WcEBC(*vG*SkKjL>Wy9*c|V&y(#qt0@8b4^5Kf&FW%5%_ygJAkUtl;$IV?H}~j zSxwaYgt)o9|H^;6<+Kt(So@(WC8R!tzOU}&@u%t1!X>^Ozxf;BlG>l7m;fW+wIk7p z$H3Cf+y3_p{1||p>Kr=#!G_fltRz((%x^~Nh~s*Gd!-qC?Ed}l=W(lEqvCyD`mR1L z?blbAESxk5S{fJ`Sv3A6@Xvv(W1&8HM!BP&Q9q<{qrKU2b}nE(*{01ec!4wYn7bMj zpOCE$Y2Rta1;{6DyRLsc2Y)6*peQjpTP@KG>XaOyT^Bx*239kDuhuDl1q#U zI9lT%6fqFY;l~ooRO4mt44H5U}Pm%@GXV-ycY&{^R6RiTG>%7@jNOP>p`K!a+T%Lv*Kad& z+;hemTTm}@4lM75z75G71goN&YVxrfw>%21m-WCy;-^EN0%Pt#kOjW`a~@2uxCw>p zXBnjKaxyd_?MtYdAmfh6zL8=3+zDM zpP!$No0}WALZiK%U3kn=TeFN{IQV|GhkM+BN+kk|Y%zZD-~k3KgU%7FDl6M;+{*xm z2?fYVThi@bpe(~d`=%`Q=wG4V!T9nB$b4!LSaSzixm8`3W$3bK@om3yQRFB8tZW6G|6+q4@lhbIB};iq#d!)ngst#>TNp$y&efPc40bc;Y1c3d~vaxlIs{?9|Ev@i{FXSUP8D@a!)|+h{ z_V`FL8}1EgFnan@@yrPcq221G8ZtbA?NK6Gy_Rby#dhx&Hva5T|jH2zWmB_7*SaLXWH^I)Nq%VND?a zKV~verJa=a*~oEI4&JlphP(oXMuP}Q;3>R^Y{U&b(2>bB_+Xw^K%ZP8!e=k^p%4j8 zrLu}T>^5pVLt&5?$)s5&r-TrrIb5t3f=x;(sn=hroDns3@qW5q6(9>di( zzTJ3>(Lc+0KJr+QFQHcd4;+aGbN4g>z_9TT6JA??-htMsu;$N7zY>a$m8qfRS%_{s ztM842gO2_6n&-{ML<4*`s>(!Xu21WFtZJ>GMsn z8PmfswzP^Dt>v|QRkT{e9s_?FRPpJ?uVKJbLw=JhDE+D!xo04YRqS;wF6`Uc?NBvI)qhW3Gu>zgod#luM4MTlC zJyBS2BGvl$wUWH?-(Rd{YGX5hvgtFMeyZDIU#=r-lRYT*P$D&j1Zh_?PCD=ijwyT0 z-#iW?aCvtX1@3RA#8h~Q)if#wiFd?5jja@|2f33ty~#fa$Le(@|L(UXJ!#ju-yXdB zZfy5im%8+0!k&Kdw%Xspq4PgKGHW#nvUiDh+@O!kM(Bi+O9Ry1B>)jv5Yxj1ht!Rn z8<4+@k%IXhRimJd1S3b608^!fa=857@1VP6Az(g`QKSY*QO-wTQk^3lwVnP7_EjGy zNUZM(ywTe^Zqnmg?I+)gVN&s9^#Z|Kxh&gU6s_v{yo!g5I^gqH>(<(g@c;83WUa2> zNrqG$4Z#u;G7)=ElAeP%ppB!wEULuT`ko5pCygmnyf97k15`K$;t>7h*5}Ogzf%mN6WH#WtzBW0^B&B?=B)o zfYM`99sUJ|h-HXgGz=DOtpQxeY7mfE^o`8MFG1Gr4I$}-#{OlJz7Q4dSt1C0Rj-;1 zsFV9J%B5k!!a%bOI@pCyMs-=TE<-{3_o;r4yH5bfBa&8X3O2=K=W%z)RkXCUes)(v zzsLn-u-ELr`uIwN4aPc`@3pJepP1J5A?8Pnq%@+P{TPKw2M9)QxDq!A0eGynGA9z*0s0Kcbh8(D?6_vAfo<* zL=Ff6Cv4}U3_Ng;^xXr8;}R!C9sP6K1BhW@VBmuPioI#13gBa;Ll#}oHqCABcP1LF z2LV4q3gQi=8g#HsMV!H_^ z^}9~AduBJM(9Gf>dvTRo2!Az^#>#?kAbn^K10tV4ay;l8fot>#x*$(Gf$R(#?Z#y) zgIhke3`aJ+!H<>BN&Wv!bl@XSEo4G2gAy|lXriWA37bSQ#Q$!e z7B#6YZ@=k|vYG^(mD!_AfLLZinH9&N6CLVV7(9>kT~U=KxwAF;0IdZEa(XX9x1)RvndOT zYflj~S5~@^cFihcYzP#Kqbf_6Y_%v?bGFEv-#wx^6<3S2*nmaGRYW)p;uS~&+dW$l zy1lC3*ZBpUJqz4wAIyh%3vaSHgK0?(r0A86o#!SlV~n*FU(fq&&eu3_rOBW2skFqP zGuN7>13{f$l^;X8&fR{5YRwe50XYiZ0u&jGoAgDK&R{&Y1^hZ(;0Va*9b?zA1Oxcr z_CR@ylgtHdoqIt-a2CY&0w{|p*{h>x+N3SK6YR3?m;QC;@`8iXg~hx_o;{{`#f-!{ zk9!Q&5M98AH-iF7nxMZ=s{oAC{28f-1h2L-T4d$3N5&+5$9awm-*6Zm!1~T+1N)E= zL8V^+X>ndG-2&E^*wxzZt@7{_!Ex$bXwL%j?;w04Tu7%go;hG?3U8L+%=i9p#|3`R$E#$|5eaP|vi9CW8GcbGa0%`W2s~-tjILlT zmL)`fj_?Edo1z#3-$1g+KJj4#-*xboQ2 zk!6Auo@gRA)a}kNRif$bAbDEGxg|>?Z}~ZHRo-Lbx)}UmY|;}(RSC=sdmZwcY?W9B zU%|_IUBJ8b6b)IRf%hA_+LVsZ^gf`Ey$Znn1ojMDD`w{z`wOcf;r&D$we{T;tCAdh zL35sbGzf@ZBLI^KU_b3E_fkMSK~%mYfreevjZJ8A`)OYLP?2Mq5MPdb=8>cG%P`1Tr|}w zBiX9I$C2r#DV1d@&K+}|*7m#7==3IdRo!4X1__r^;KZHi7@hlGGsXFa+t!5NS`UBs z{7LcnP}@<3EEk=TP+n3zWJ}?QG@XZhAgz@4H-%dvpa?Osp3wjpNWXTDy|_q~7ZJ3) zY1PSAA?A=@JQA2WrQa>@c#RocpWM`fX)c~)M6_SsiGjso_LZV9wBGJy-x zcM<@9^P~pJixQ-FdevT9BuFup%%$GR_Wy?0BPDzPb3qd7rBp2~9LS71&$ho4zL{1Y zKm{+m5NdYY=+%oy!h))PeBnYx)y?>FrxJZ7&bV#HanCV_qQe$0v1v2p)sPE2$cLxu zI7r%;eapLOR68D#k&Lc*NAn}gzg)`WTVUi_d3y#w%f@C5td&q0!&W8ZA5WG2+xAt;ruw&wEi5f&%~|H)`e+MZRJ;Qs zox)WPP#_5~4GFo}2;zF20wgAu?w%E!Mq7JnCtri?>cwB1(P;9M#sZ3pH`;J4^n4}F zM7ujzZ%1%$B-o|4`$f1xZW@`WO2d}lBGzYw4a+~a|GunhAgsxhoAe{^@Eg7rOFi26 z>J)S|Tp+eh`Ky z*1FeyU)T4Ve>W2U=Dju++Ra70BibM@(4M_Z@WI}cV_$U%BV9MKz(G^l^=kYx^Bf{_ z!uA0r@F3ecf%*+ILf1#!1rl3~Zdp>dF^oG2?1p?DK032MMHZ*Z7MlbW{+NFy%{5I0uO?T$^qMahNEqMUHqE zhSo-7Zdjil-2rEx4=TZj0ul7?@lL|F3$|(G-vR{4^-Gu{+b=c=X5orw*5%o(5(lHG zBl$6X-#jnah)bp(uu=NaoNbZ*f0~()|HK(%Md+SaJC*?44yg0VsK3lBHL3VtV}9?I zUPWP_a2w76j&V8M`;kO`@yu8Co$TU15R{3{%J(3+J$M06O+SyL$^1fo5h9epwnXFU zca2uqzs}gfhyU3Wp;Chh+IeNQSoF*9iMhP3dnxMVJw9FG3?k3877^ z)3|vFcmPujETGA`SOW+G(Ab@%0+PO`;3tjX`xSXX*??yIcpXjnf5+kTK_Aa1ztKF(^uF-7l#_(4W z9al)7Wk|S1^S+{fg`;w6;oQ>5ibEXWF?!XSJ7dR-1pVR;slv+eWk@p7Ux=8qRTX>O| zKu;r(Wg!8UBRgpIBcKEM2hu}0Kpq@Y`3_(BI}`K8p9@w5C>Y+xM3;=A+K0Gza~yCC(ZJn0&PV9R?TyiG=qgh>Bg^enL8 z<56Drm_5@6;sfX$n7?px;}P7Eh*@zEY!aC~Chb)Y-$=1KTnHYrT+$ZXg0_ea-7*5D zmtNFet#*OFuUpCn6#xSYr8?>y1MF{y?iBby_&)0i$>9obXuY{A?p}~UH-uQLc0l%- zf9=(A4NhGKgi}BzlQn79zh?S@y7aaPSw<{=F?c7R#4xKmoJkYDPIp@f{y^W6C)fJ` zf5F7Mk4e}TBoA(jho|zHg_EF#(+|d8;R~MK{gsmx-qQtyFvxus5eAiLW_U5g(6a;{ zk%;fOQy3&^Xb*oZyiMgtlKX!<*$S(iG*+fqY7JsptiD^iK#z?&C6q*0_mWE5io|_& zbFt)#RpD zgC|ECHyv#0ZcKwFW^2S;%9A=(U$*N+S8vVg2qK)RqQU&K>9LvREdHD|syZ;zy zczvPg33_I(Bh!7n!Toq-R1rQ7g)qd?}3WQ>j0R~qN7Sp^&!hSL9xGNJpe{NH|x?0ZS8y+Jj}kCl&b zHUcoLyhe#f+@Ry)^bwecWrFY{uy!?4u5>?46|mAYl2BiGsuMx|*d3(P^}bZry<)OUqyb13 zwr{p()I4>^qT!Q^F{IJB6jqt${W1jiE|FgaqUnVSnM73Qp}ioECiKm(9zkeVfc{Um z!5&&){J%W`mVo(lkf8^#T8Uc1AXOEKG#V>1>g_WT91Y*=LEYfv>N17S6PjvOWM_bP zk_igMCm`T7;`ndixr2B?Y~6pFO+aurFfLce;s1>{Ec~{F3-~$F_ri99LG}9th(Ju8 z+U~xRy^R)AjrR&+L=s zDxTr$cOo{Vd8Sg^4B-h3SoPiJZy^mFdj$Sxpu7xucdveXLRNO&s(w_rW&snJ@|GQt zbLcF2{@-GQ4<5>YE>u)Aga}NU0L!C1q#>i807GoG6yf=i?lW*4&%HML zpnty-eAnQAzv~}cNQ;Al>T-Z;{?9EW^n#iu&>z8M9rl0EG+0iR{_{irQoYi(mvaI= zmDLW(3w!GS=-{Xk@9I${cpt;Vu@!PSYCKMzKsQ<>^-J2ceb5xq%_8%R^Kc5UDzdQm>OfrzeaQ^R| z@tu5%VZXhWlkPHkMQN2YO%k%zFLT>lfNZQUF20oXFRTg0J z04At^bsG1%pmzeV1&}oFlu7~S!OA&5v?i;}0Jb0BRnYb`bwy~z1@y4sv_lB=C9-Oe zfkHq)K+fmzmCs!S`0n?@xTW?WdlIe&|4F*t5e`q{31Gbc*|NU8XYC0%wi9OIvVh6N zIRl!P;3Dh;0dde{f9FTQE6PL3`zFZoZJ|#2_a}%ORwgC3V37b4$FU9`xV)f0h0nZm zNY9Nj?IZ|&&)_X4WSfKNxiMX3|K(t?3VP5ap?MX#AFnpxH$*t~d=mKa>h{OS)poeZ zYaPJkIuSs0d;WrbI18`x{W4);zpUK<8(FfTb%;#^`e+prp@TLnIB)9-p9S2*rFVkZTP_R8+r_Ky zGvE5202IlH_0^&f(~3>cn(uZW@&4p3!83|9|O}h z7pOqgk`dV|5Hw^$6{9ts7=q#nsA#x88MOX}GFh!jgqlC67yR2}51JHQmCgFnzCYg= z$X^Asf2OZ^CgO_hP2KjSyV(U|7-o75RgITBx`8G!^7#iN^i*y|snY8!94vV9T&R*Rynp6qW zc|7?Jw4DmPx{fnp+C_LqkK!Z>k|_-fk{Q!SB2z=ECs|zSLR}jE5TPrqs4i9= z(}P!3>F%ngbMu}`k5&*K{`&hPUc~3Plt9b?*XF;*ejWv>H(&UC!!cwzV0Ct&L+W}% z|5Kvoqo82s5+bhAC16iPEtnBwS-9X!vnrb%H)h1)n)GV@$CCJ9$)!L1je`8Vn)nxmi6gns1cxL)%3y+n_Al4e*116xBp-9YNiHKR-5d0vRLVHe-h?NTd7pTNj$flQ{b?!kNxBfA(q zKGeGIgls3E!m&$@%!1OVFg+?TNhB}!BJ|Qgmzxfp5{`lYAIwg%Gw+JDn`aBp~@ z4yKEyM-2`OT2(l2Xd-7psoG0W4X^|Y`Rl%{ui$WLVZVWk^d#;YaqKQNoti6;ZCO%& zth+faIp@#68c6aX)U$RkN@1DM5anD{%XEckZcb{JQMnNn`_@^gcGXcrUACae98Wy28pS0)lUGZTeO(O1xRV8lVNZ2KP4*j7JZ!!!L9`UsP8dF-x;CJIZuU+b(bC zNQN4ZUKe9#XIsdL!S)>@e;m{K3?yuwZbRE!D2x338 zfUx(lWZOa^d;9k2SF{&YwM!x&3>&qq^ZKm{2RcQvh)L#$`{h^CRmjNwgCrr8%t@JD@>HkjJzGF#u8}-xzYMm_6rc-;Sob5$OaoiDEZ$mSxGag3%pUhg z{>S|t`wJaO%<`viXk?Jxz=cmdTExG9=X0w7;3`#G&pUO42fw2SU&8ymYVtNJU`f9C_M1F@pCXwMIbsx=a@%EGTX1!OrKWAfjm|_Yzeq z^CiH_TR(A+^?MfxGYYvl6`RH=lXH%md+;y)L}l&{PtcX-`6W#tJ`JU99DXVXv-g0J z3=%t4X$6bzmZ(!=IkuX_VeG+Zp}~%)sH)Nf6}45*y;3qLiFHqNxoDp^(ZcYnKSdM4 zm05o6b2Ny(`gRj5{rpJzob=f ziu>GYP_VmR?wdHrY7{*U_|W(VU2lQF`wwWAAdli?M2<0pJ6hGY`H=H@slV!>>Dv~Q zxofh5&Eafkh-2{paf$6L3~2yaR=_&RlKnc!x6$YPEEnQAhxA&WzNL1cgnM_sY`=Py zNmogwhj3}DFWffc*|ir6!~n%nm?;^glSu1;mIO88sP*5qR_g@VO%@{4PV7sq%gW2E zi@^A?H#|IC_%A=yoI7jT$Zorx)pWE4;th*vR(?mcAe^*LARc7DXXMjf#%aaa+J@@t z@c>A~ivRoTm~&%iMRxWBd=b1+qx4~#TJH%ofMVgLaRFND`*oAGF|oK~_*mM!&x9Dv z#dEBlXH$Z9=61u@hpsby&JoUsBk-(L&`sA{$_YRl>6rVCySMJd3}a^0ZPtmSaJsS* zm8}*}mpG6P`|}6{vN(df)@A~&l*x76QfS*sBCT5?88+H^twYd#x40$%r)#Epuyb^` zkgHXSF|YP=^vC6t$VP>igzlt|B<0RsFC##~)N`NtiF!tT|9j8>iXH&cp_P6PSdZ;t zUj}$W?>4L492w$)^h&nQg+D2xO+R(YBTB!%#$x&)-2Snm`MfA9>+CnEaPx*e9IP7C)3sojKEz`8nw!~Ru=Px9PV0VEz-!66S z_B6M&9H>zS<(usL%cpt(`%rj=v{V)&#{a>@17!~ ztu0ojt5i)_1nJ{qX0o!$gt>m?6KrRk{xr}eqgw%cL3~y7uVF3dZVb`mt62WYh;N0ulD5*xO((@?PF@g7*TwoAuer zSAXp;>>x&hL}8QsWN-p{F*TJADtC3;^K+zRL}jM#hSRGM@r_z!fYa+_dq@KaHOM;$D(m}oi<^_E%C~~S@6jwJ-AiAC;&V*O2oc^mv7RcD-2!KBnm9(SWoiuh~}3nDt^8 zr66*VpFbF}6%?%yZ0zr11j>*yZeGzBHv^M!8etlVN zr4+(k`7tWsAYx9cGQ?Whqrr7`elMfI#nc(UT<$iOSPbI>wx#P@bCs#)F-#&8(T7i~ zQEsT^gvI=c7$-?WTfA9Hl^{GF_S;rgn)i3E+Nl2bPWU5EJ^$|Z7z6`y)l~E8XFBRL zy}#4WJ91De;om=dKCVXK1yb_!uC2dg_&41YTdFKuxP9Wq$2;!b#0O&uuQq?&g|PnU zC`A&T&L-xDMCW?r{(AU(qhbdJ{ZmW(!yCrz=@tJ8nNO*DJdDxQaboO!MBMAUfdo5N!*u+WVfJ*4XqPkLrk?j zK}KonYq-pS!xshJFo|S1H;kT+IZp?nACzjgOkDEraGlolag9npV;C>C*)68HP&I?2 z=YjJ-Gr9>aA;p7hq0Fk?1c<$iIFt4c_B%@#l)08jVQJb@?QT|nLemH^4p1COCQElv zCts~HmO54U{hM?ff?d5T@dNk@mS(Zi8Xg~y6ys|lg)QSCkJUYQJpl!!+MWHSFC$FT z5kN+Jvg`K4^OSK~`b9FRUF4x;T7Cd6Ta2_RtvVR3BSB|%0rx=^-6^L z^9}lu*nvP<{(Q~k(h)5fA!*F#l!qH-4~V7ru?*T;OU0>0oql;WBu0+DT8+q7Rnu2A z;8~Si*3yl~QmWH*abFLMQ_U?PRr#B9NrCy7z2A)kPzLelq6SS^EEPvX$A_zd_8st+ z{tO%1?d4tVGEYWBOB})F-}8|UGRAE`Zmn_N&`qz@NT-ndKst2Guv&qdWj8AlJusOOnQY3Od>H4gQX) zbU1jd?eKKuXVKZSO>b1JhopI-_C5P7Bj=!sjEHQ3d``b|7PJYdx*YV#_m@QpofiuKgZX#k1DRtVaEvVa`@qCC+G`#XJJh%qd|Fv{>Oi8%mqxlwXkxNjy zNuC~wXmftPsWp|diKbX?e7_COq=Vuv#~iZ9^#Y~cG2iSWge0??4K%%{@%wB$r_Gin zVPli3lRkfmVs@fCyQ(NJBu+JGI=yaePxtobyCWa{kZe9RiElTr&_wr1sD1-w1G}BK z{=KcC(WLv}?P!&1taOPK)JSsfwf1r3_ya~70rYm#aOPD_W42;e? z+!3G`yue?!CMqW8QN^)0pnfu6Z`^Ck2z=9jw0kj5D}y4&{mZP(bL>4m#ASdhD(S&* z2NUY#(GNNjgHuyeUp(#jBtmUtZ!v4MnAnw7A85SQd^oyi#>+hIO1-A4Oj5amXBx)3z%c_JJouFFsR*qWaVvpP zN9FZdX(P80fq&G%K`LqQ)>)B*iYr1lp|-3&rKs>G#u30FZ9lMNU@qxJL3H%Uho1zk zKOLZUaoQfiVAc{_I(*=8#u8W@eq3VOrK^?Zat-x(>F_abg|o@k-huHUIc^k4VN=p? z(lsjHFv60iR3=_>j?0|#eAeC$0)BwGwZ-1)T9RSn&BI{P=m$0N4w8+cB4g>KT%)Cj zi-RNnPuN+~cGHr^+1H05lud<^^xI;AL+b{vsXKpG={-we52aVh0q^*#NlTVUlI;bz zPIf5zeGTj=Hq-;tn$nS-mL!gPt(o<@9`sN;A(`xgrQt$S+>i3wGn|3wH~1s14H~Ut z6QC+0*~FVbxsSA~kdVokSzj7Q(%$rL?c4RhRwthq`*AMPPHw|tASg`l)GC)^-|S>CHQ|9ztSeLxL7g_%Dfc)I)KbwCG6@a!jQM2PJAQuWf8m)lge1H{902P{>!3K9eJ+of?)jxsJU z`4M=G*xsJd(S#5ALenkLSeHv8mRz-=C}S(iI0s2^RH3V~y6+hQ_O|Tg1G}ZJ0=b$& zIKLfiS&H`j^z8=36E;v8mp26hn4(U}iG+`LkUGuGpQ-7qsMeo(fWYCJnm% zm?h=979P7T8rxJ=Owigeg<#8s&B|xRU|=HAe{%D&fq0kl$h+ihsZ^}Rg$w5V0Hu4@ z6IWMK(HI4`hqca}#Kcd`?17vSQmS6}B@`EOnp9rw_n~wDviDeCwTLMb|G``pdmP&! z2a_Ne>N^L-m#e>!J6qBr;NyemVug_c;D(6f{CXD*~f$GY!dz z8NDYkirGwdz@=H7nNg9LS5X949d@E-4g4}w!nkG%3&q}|2lUtMJ0>8J1S^)Su&ktk zR(X@@%HZ~$2-_EgNvu|_3wfUVJuT=W-khFp_qUVL1oaP<+=SV9xkk40ZYJoz%MI=8 zOAbKYpr7i1e9$BKCp625dW!j*)n@4L`!x%F2^qiA-DBhYHLGpy0CGsIrMuLr%`jOjtH@i17{;6A=CBAg(+)X}c>u}FVl z?r?{X0_(EwQ`p1jqB7s;T*MFGpT18_!!%tRE#&X4rt^={9FYB4dJyF2w9G@-&9gM# z%aVVd?oyD1AsN8BnY164scK_5GK}eObVG=A zN!n;Qe0joKI9xC@LXw16@t)r^@dGo4`=Uhf8~HRQA3imSR1hRM(d{@3EVW#};g`kHJL0>CVvUApGJf}ntBw`*X{i~TcBUgzHnUR*ig9NPMneWnSN zt-yaEF%X(JAS=tu+ZZGmQb6mmRVqs|t*fbXpnT__#DnQ%f;NWCiGLb%?=QX+fY=Vs z^N|7C&JRZ+G+pj6vIx{jMTSHQsgU(&&~ZFzcynHc+7eYp%#OetX46U>TaM;H$)M{- zJikYVXo6iFx3K$o>r}~W%m~jC5Oj5dU_N9Z3cDq`GZ{Va4vWVxNDjyjU6g6wh%U&g zJ01NM(l@c9SwQBqn#Xyv7yate`<|j{Oh9&D$AUu0?61y*C})*oC+1;4%g<0=H@K6T zv-e=0Re(BrY79hry&apmlK1t4EDf!?ukdjM#k;^WG7R)V;A>&^s6Xt9J%PdAV067H zFc?1WnV*l!%i})#I35T^l<+L#lNYewfsXQg%y^~YMjO8|&b)q5WL;UzMBR(DQ|c{V zF3=Y;c;DMXgDXI5;hR+#V8P(s8Rozt_@ji&ZaOb1ssrSPc^5AevwzgKrx;@Eq!;h# zW9bsMvpr}vn|;};h+53stm<=_t%_I@s-u&lP9g6dQ)z*-pJ`Ehp?g8?xb(B(KU2I@ zJe6g`BRhal0yDErS~rl_lrlRSVa%}qKA5M~WdwyQ)_XELLNDN0ef2wH8Z&;-s5P_U z=f7@+AJwNY;RBANIiI$Kx}6@5GQn=r&oTr)XgZBzpiX`@g*&OB2pM}gRoiZ|Hc*%R zN|t0R?I*RJ*~n@-&2+JCBzKYNe!+2Eq>-4oo?LghL*eZEF4D<%@H6h##WPos(H>Jp zwt};wiSOmCxZr89*K_iz_GrRtx}N{}&seJEdx5Ve@(TVtH6*;XDMJfHfTDnHuBt`g z+!k`z2IATD3TYv5#tH@2prT-2ruHt;E%o1s@^P;}y(#7P!08rU)i9PPDl2%Crj=FK zK#}+n(KpY>53TV!n0~n$++^zIB8T%ja86_wz>J4`SOn?dTS+I>7@* z^0XZjh}v(|K)&R%9Fm@JQ9~C$QfEcW*RX=mHjfDGfhM+@J`qx!()BML(2a|pl(u^qiMw5?S8WRzMGJM^$03no;-tq2Z z&yA-yGWc}W1lgZYWKWsFgQFt?aL8+R#hnzhA^ia%-JocI*R`)CW5>dK`Hrvbe`dxrP*G-fH1&0$eOMiO5Y4QQC9~G&c7*y;o`wDLAP@dXFL=qv;Q4lN)bTV?t367+N-R?5?jPAu67Y^xbg@&@m7hCDj9S6 zgB_WEm+K~S26cR`>EqwTD&4TFgkESuTTjrOnM5a_qN3!tqw)>6#cSRH3_SKUE9n&` z@?p_7{@uG5W9ZTMO;L0ku%YO> zgM#;eQ9XplbG||TiNw9XWQO>Pv0nA!KJDF;lgDUkkDx**j6PiYP3M)K(HMw@`1UfS zpL!;T2{|j)s&-Szzb$LS-=dj8ly{PsR5>X_Nq>v>Y{4OylQb$HSMkfC`}075l(7xg znXhLwtTVO@fm~v(?0-p_CbC_bZ>{raDfmpwGWB;$WWmjsuny@U*^`U_dH2nG#m(lV z-kvS(0lp`d04Tc(^y$u*2QrC6hS&5Hu^j5|H7CgNZ0i0}BH#`|kn3yc=mRmv>Rtz3 z1O%N#IJInM0a8QO*J=G4i%1r-HvJ;Q$4(FkAp;$nl>WZy{Lx%n2uig(${pV z{*MsqY8gj2-Mou=SMi**4ccQL;=mPelv2iMm0QLoM+NVixzru>gKGVpT;Ht;RsIIZ zr8x5;&I#z_ZFkem{qn<3>fY}r<_1w>FN6syZD;ENgR-SZP>xzoAyv>?M<)3562ike zG=PA~{GmFM@CO)D5+nZi6Zqfz1+B-ixP0ht*}R@eT%!?|v*@=mX7&)}w_yN>7C8?s z(iUyUO`PH@6nJo)_#4c&6iJXKk5u*F0%^Z#2D`7t4d z$AbgllMtih;UdMsBje1pBN7*M<@@&{TFJ>hiTq+pQThjlBhKKCe-n_i9_< z5ZKN;Xx;@j7t-c+(?F{<&zz*)@`Gqy&5Ls@eA&`Q@KY>tHb#qJ?moB|>jYXJ$>$@_ z#nkv|-Pe;OJU+fjlKV@uj&bRE25v+4!lTb^ad>hNrMO4*ePJbQju4?J8e)LmVneZ%^t8 z$tA^x$2#LafZ8dXMJ2}d`@u3nlOs_7NEAS5Oy@job^j}Nv@E0VYO}+&It;tIQ6k|O z%yy{Ou`wJ1-qyv)_V$q(7zoy(9{5uf&}T@tsJ_ECT!T*#j}1F_;-wL>aLB|tX|ALZ1q z6NYHDx^7s=;sO?=z4%4~6cUCVlww8`4e{^OjXD;_Mh%u!eRzb0oPv@S`*T0eq1vfz zs}}R9h0~n)Cu$0%UqmO{)ci(sqI+h~RjKc{Hs10jAuh<@3i^II_p>DD_?g`jyl*6aQAIPFo}cYPTbbrO-?J%?+;o zuMPK7s6>;0DvM)&yk(`3yk^FR09qC~h`lNCv1}Uzt4FNm8Fw%32WzG@CiNKb4EwYp+9u+7&$p>=1PVLOGumRsjK zP9flZ|Hi5H?F-FQDv6yP(GCW!2h|R;?EaZqRngSwc=Tfp9VL4UW32~86ak6_y6RC5 zX@HnWm8B#urE7GqT`*;Z%#MAxSgUbUwe!9Ag2sfnI7q(OR|!~_(P$wDiqo=U?}N#II$u>aI(JdX{Is|YkJ|2%l(I>`5UF0(pjRDlh;3z-Ou zLLj2*)59?0#0ftz;^&t(zNcUKLKIy$lq=>`Fmn~MM`}G$mP=p7oa7Cczw-?7eOj0p zm=S!q2WKa6fJ_Rx3+v$F4szwfhz1lBFUw{)S>0<^SJ^>hYbpjq!0Ig6l20y3n=CKW z+k&pqtjc`7@4(YS*Atga?`LXd`o#d1E2(%tawqSjjq$5bI01~9BH_`DWh+IZL@a+M=zps_XS@NgqyT z@n?ywTj&$U|CU6?)x^?tJU2+(oDa>Zn4((4PcQ5r{vPoH(%~^#N)YF!(8=%}ep-18 zw)mj;Odj(cYPH^}r^^t?G_Y!xuVIIJLXr+Keo*5+roS*umlzBpVKjX}H2{EGH^A^TC6aUeAJ6k`E zBJP$W)0DiLgafVEHv?&+Z@P%Pg-mZ7rvWo#F>;@QQ#MDLvziH?$m{xsui^q_xFYv^ zTcg;9p55$-Nm;eLs>6Uh?e8P$i4X3K)HKjEC3rvIr){nmXZ}JI`8$T=5VLiSfZEWV zd$*-~*m5onT7>kThZ~mJOp6NDJnBJ7c58Eo4s<8%`(1#&qut~5isKRSS%-rgM-P|*Z87)UKVA0!BW7~yx)vWDf7HLnSS$ljLW2=blaoYLxGl~pv==(uXGHz zQez!=lyOTFQ#icO5-~249edqQU*xbGvk={8a2{|lNYA-m+t%lzrKvfxB!tHnx~qG^ zGKIN0epzMguGX^su`TH)YT=$_k4sv5&;%OxjbD0tzB_E0DQQVat z!lJGAy%=*wbCd5kx#(}E6wslCnNl4J)H_8$_ksOWJP;K?h@z@RC|dG(XO?b9v3;SH zE3gep2t~Z0-ESALvmh0N1EGIF*{9dl4yJX-{1se%Kl2}Cf0fLi(GmfgV{!%;?@{HM z=}UH3ApbgHXNWF70VW1;k84CHS8yo9O!Nw`Tg)~sX3 zqTW5@(M6qk)J1K|`B<@{4Z!T|MM-4zw2JYpQA_$5)i%vk3Y&wb-ejH6AiWQ@46JcM zfsHGi1y&orGce)d?f%CFr?1p2_SM8i*4Z^}(*P5LV*UryG7X#>;fl{Bw-W2Nri`7x zYI#R@8iAfkaL1>`cZyquc~+*we7&V!7JcR;KU-1GvzM5PEk_88y$v?|345~mK$@5# zm-0F@2_YAb%h#>jJocx{$l5ZG>lVnll3ry-b^^z#18&aACwc_wwhduD#k$6NqFA;TQvr@H7 zlP;(E`nTa^M2r>7&mWJlN0-o6?tL2%YxDMVZ;o*87CXTaHockGu7w3D%07NEZDo}c zVEuLXLVCojDGHNuZDLcc_KEh{zyQ|r)Uh#g>@dWrV_9Tmu4cZ}uun_l!ZH9#DN-^X z`li08X}wyg&In9%3CZK5heR|#0ZINYpLWML-eev(pd}Y>piHC~t8X+f^Ex{c8KlT`gx69}^KHGlvVBc}a{i9J zs}~{pNL#Pp$D%$Rznu&Ot;0orB_K_+C#tsBsI_)@0sHQ%y1qZ4({mhpM;!r~Gg&9q zlkqVaJSKY!Su@{#qoC!mxuiKC)II^l5&PXr&j%#3ICUnG#T~Sc-dG!kH;y0JiV6Mb zJwX{s!j_dqCLJG(*wJ~?f*j_AS3vrE{;vJ-?N@)b>cegW=KJ}26(Ez&mCM(|=r~}5 z&?npM=6=eaxVk~!c6Yj@?7iN>@+kI9v7B6;?HNyBIkLE)=%J$M%e9f8)%% zZ(I9B$9uE+WxeBLNLz07Qf}FnIFt{nH$JrIwjWd#sunMNUCD`0bl7~Qr8~}uqX|D| z+6gKQQ?qRP@OyQ zHd1r~K7hpV!*V#nq>Dv0_LHypYuq>*kd}gUa)`6JT=EO-^y(bYw+C-8{2tq;6w6=NG*FHTCVyyix&Pqe!clP zv|PZgQoHv4nuOWbwHZEl`hu)uKIGYa>)Pz=nL+nLbxvJAR%>^9YYXc}CBtL--}XD3 zF#om6-CfWX8U58MbFVUWXO!xQWty>pgY+xLKPyX_W2cv(CwWC_ReT)9n%Z|sm+7qY z;X6IhF;iufAol>0f}Vqdd&1cgn=#RvgO$v&V5nQokSy>cvjG4RExeVMXA59l?DCBg z1>AX?WUByfUKVR7&xxSitKpeMvA|0xF%Eew1ojua^&5;MtQ}x1EsVd_11wZ)o}K(V z>~!E*@UaV^W~Nt13kal+CHbPwUx!ifwf|%rFt@O7jk6I!( z!`{0huQL~93?i%Qt>88HgGmk-{ZWJ+7aMJBjLT2{di0j9So+cP6K|jO+?*6Oosi{( zB)ut;A_@lvEr<8%=SwWT1d271VQwsi&))isJuRszGHwfL{p)BOOp@kxq>V(pnQj3> zdGd*Yd0eGr3EaUPHCJ!ik^(an)VxilgW|VDW+$34O~IIRV8X^L*Q|ofXF?!9?_q73 zvndspxrYl88iFqa^sXs&f<*lUI4KQgI{LhP+{J$+#>Tj0dcg`(MyPx{{b=+i!o z;YMqhssK%7QX*HfHHO8kq!)?dn4t7?=eG`(AH2OHKW0wjWR+P~i~SW?ebG za?Z4YOXq!(aZs&aT+`-VkIfEinR%X|>!3)jLAP6X!iSblfLjC}JsH?r7H!Z)dsG4d zyl`19L9B-&YGjh{0t}r+(<>k&3tz+SAGbzZi32Zn%Q{NM_F8xkTe_RZ&C<8dnW}DU zTkl?68B*T7ov3%FXL{Umhs)j(sNDk2wCX$+Mu}Ari55(v=O;$TgX^{Y8KHKU!V9{f zJ;KIEq8TifD-)1V_LHzwWn?)3U*Pk>``&BHB=#mBx^E5;c1rMPHw%76%>jOqb3G5l zqKT0VU!=@X6xXi+vZ=gaDJ{#>m^uBiy(umgYi(3gX0&z(8Y7EmXr*D zIN&iNNd4=1Z9i%aC!@{>eeB;CK(2q!_HYS|VqIa#oX54Ht{giCj%`%{rv=W>hF`HE zh8L~Z6spgM)>iHhXyFkbJnweWoAUX4nB?OQuuEi zYpeR_Refdo0KYjmpdV%nGU9>B>}Yd*(Q|>lGUectAW)%liFe?(3cGA-QQxN|!rY@D zSwjEpa}+-#9ChOgSQ!3MSie3KQI*vc0>d)$B=;ZPzkN7NwbVu4)NT407TwD_8t1^p zV6o<3w@Vep)Etj<7+_1bPO!z`S%2GtZaLsxLhM)m1pc}1?O=m=<{6~kdiX&MW-Y~E z59aKS)A6;9rxOM^Py<`3U6d-%kv$kVY-{O+95E%&y@hF*8S*Bv@)%zzj}#iv*&aSM zTRrc+NhkX4d=W@D7v(7rnisy*0=Sm5Y}hPeVQbI>{fiOx$7YD^;JQD#EuZ2Q`{@@0 z6eqR*K@3AW+2Oq`xJe`opm6yO_mFq7MGEbgp#7XABxu_}BJ}kJ0 z8^--KO?=IBJ<(^!v!h;P&XvR{&dGt(%0ol)ByF`zC|Wxon8N-fW?^%?>ppxNnE1dS z0=&7JMTCzX^ov-k%V z-E1EQXqA+ne=7B!GX~j7L#Vj*A6Ygm*&XXH19=*{99KO}iZh;NKmIMo*xpCsY%9%c z2SKJIx%PkL$N@($OsiNp^rL=;Q8Va6eGHYUX=+VrQ}6yrBPIy@K_$u;6W+3C(g4O= ztAH4nR7$!UG}rb^L^1S7LJJUzPInD&--cZ>^73U)G<#|ShKIgNVpNHEB zs(8gqYtVYW+BkNO|7^lo+7*D0w~i@J(swG0RW7O(0ziQNhultO!JdSYJ?yO3LK9Xl zc@li+$3NM(o^f6&R=xO2m;(c?FEtd`AAU(g9gF;4rtmlKuYiBOdDygmou*YcXR0aI z9Uk>nJ-7EUgDi|NpgV*ZcFEG@*u!oQ7qVyf9vf}E+$tDbmSLlC!=kqJyMlc zXC0b11{!PsEST1g%|&FDy+N<2GrJ#%L@JONN}G+yH8zLz)GtX$(M8w}H?C7IJPh}J zZE+m(*bJ^8D_q6QeS)JxPvV7f&^$F;HXEkXF_$bIP6Ht6oXi zBeN;gyW98s#zyExX@9?ygxwV*cl2R|-e^Qtm-1+Y3$+GCP=s5s0Y4eQ;idu+z(eud zaRp-1{5Os~ahYvu)|eBJ&Ko0O@~;ma2|ocA7-O-DS#Qc(a}6vK+%cfoWJ#+8=vngk z5qOEC@NCZ5VdI3mb_bJ@{gsY=?PE4uSn{Q{K@fC0)p*$P^(*se#cr{%@Jz=UVLLaq9Gd5YqqmketfZ&sPX$br#q_PPw^zl1Zu8O#{zu3t@rv>!Rq#HI&p5u?(=IsRW5s;~! zOd9Xd7bxl;)go{vaoPr?wNCUGd=^X_^jdn`kd-4H3yC3FAAjIgsub6_@EW2n$SpQ@E6d+^7Fd|R3=3S}QP&2ZEi+(#4k-5zFn^;(Mf6Ey zWRCMLm685O!9h^ArGQ7fPgIgiP`OhjFVMe~ci;G{PVn*B>}CBV{I8SwFazG~U(*T7 zymEgB%kU?T>?<>t4r^)`Fa&ffb2hznpW0Q+AV+3&C+utv&ZIZvDaWX zD2Rd&SPLzCX&2HaDa+t4LR|GPGs-FUPk;`q7imSc~rT zPfSocJ2~|aZnNWDTPZ>#|2~mCjq+%q1I=k|lgz?-U`cdsaMb!-axAe!ex&`{5a=zo zeNxc0ycp5iD`kHsCTn~69nc(mVLu6HBrl3j&iibQs<&`w$l3Yiyo+h&PrExwwa!?W zRB)CbQ_Rf*_~)Mbbe>-ANVYv_n*>;KKM%9A^DG`W|9EZr%~p`c{6%-lxHS>ML3V7C zWG#WR&Cr*kbQ~+z*m8mnZj49|btS5ABhL)FbqjYWb3gOe7NC;a@hSYjL8F zK4SW1_KZ`uLX}5%K z#oHvdC;NIKYyE^1`eoJc>t3fo5(6Vli$+f}>1QG*D8&sohN)R(D8fi@0kSJ_0sAYc zyhnb&G1kYLrPmxKBt-8}uVb3N>EnG?sdAp}iC$C;#r}@=9i~c4f_T8zv%p`~tLA5- z&D_$Uwls;!5c#SBy)BvSRCb0E!ot}QrYC-uX4Jvu%$4T4>-gf8e>qN!$&h1)!5Fvkg(V5#I+vi) z&UM~SMBbo=6~OS<%+6YjY<8wRTXz&O`^ej`<>(_?UgwO(e^yA6Blb3iL8Fhsp2w4< zJ+ZVm(J|r-7en7^UMRYmofyB2=6haxqIl*7l2o6oNxa0$nSMNOFVU*(|E}ReTjF6w zyC^aSe_PLu3)jE64sMUaWbjhl>8+QhR8)>uR`w7m_V8spuDu`nu8Zq{x;ES!KxxdA9d+m*u~|b#$UXt9Cf5C-`;L+yQn=MzzWa%>Ro9;q^=BsS?Tt|a zhRNL~N$nsVGz*hP^XH1{_at9~i6zO{$rhzZ(@Qwmkkb@F6|k^@2s5 z)kk}*d|+T_ah8M{rPjQUr0c6lx78gGK%dI#v(oJc9`q3upCiuB&&>Bu*6s{Ol5P7X zDgRIt?n(~W^TFC1--|nODXx+|bl*x{Osze-;Hst+>7LUZxM@RtB$j;S@td!gFSPXN zMgtA($bMZ?eO?vbn0so3GhbmRNi5nnlUER-MslNSjfEZM;D!VpdGj#_>q3 ztD@<$N+Z{dq{{oEq^Ls5t*)FgdO`zpr?{-fa_{Swn%`ouP;j1C_A_<^?KjKS#AA+I zCZ8y+gkIk#zv%=%p$X70}UaDN1etkAY#kW5liV0z-pNn*O#C)b9cTMTpzE?N%X45Yc zz4)v8bbh{vjr3?Rz0dxCt~uWA?-#ILTKcbMw5pD;MlmP1n!Ro0Pktg=FFs|UEy%0m z6_iisZzYBz-z=4;$WzdD5pbW#RS*_fFEu|BTLAC-n3*8JONgmqFAwjLy<-SpONOTN zo{7lNgK>(tzisukrNoNNnWiUOQxW$qMa-_V%zZRFVhAiI89OpN__0hhrYrIZJ=f1& zSo3!7@!E(C_2JiWYgt+Cqb-qX>7(|>nXbNn-o&B!+8BvNf_}t?ALOnmWv~Z#_hiNI z{?1-38%h^cZk!o+qXOVcQY>5-T5Q{$Fc+_RtYh6&e z$vYH(v046rWu0f$i7k+t^)`v^r+SsTonJCUpN_Dz0sQr@?76DTN-*p}QbIj@Xs~H) zq0j+e*6M0}H$mEeQ>`xQ;QG=5(i0vKRY&wStl2`6-A8VCSg+zpjp%$UV?gzXD}GTpwuNPD+Uj5z;O4 z&|B>i2)>Ejf9$XBqt_+UGD9r5AD{=nE|{Sy;L1MnnyR-~O$OOl$hZABX!f_a*%phs zK6=7h^Uvorg6)#9E3>DkJ8tT6F9o}LSA8kf$(;qbIOw&^ zHd!8VMDEwvQac6-*$|&_DBNKnUSZx5yw0(aHty7}63;V=sHB%Zv^7a<=%tLbrg9Yd zoYp0=Te=(Zd4hm}>K_zQ;tiSC2^ZCtUCggf>qhJ>;|`J*U)S?bmvmixTA*Atq&<2) zZn$-5Cijqq4%2?E#Dg-DK7e8&EQ0)+seR0pv6po3H-%n@9Yqpc=K8%zjGgm(&~3su zQ)G;=rkc2*e-g{NU5>glf}O?zzC zn0VIt$f5CWFx$RMI63d|hBnV88f}qUW?ODe>?b*8*mTm zdmAHfIZq#?MsHfytDUj9pJ(J>`PZ}21OG3CXkZ#p9|8dhd$kPk8CFgkegmw)Ed(!b z_1Z+WlVH7)^J0?kH@6PNkOYVTPan%C0D%E?3nGB??-0S+wA>8kn4@n$@4tyDK%O7Y z9@hV~6ajP+tCQHt)ASvMIDxrSMgRlL`@r@UAYpj^2y}pf={N|cv~VH>@@+v59X}YQ z1~Ey1{Vn)MW*(v(Y&g@m6C4#!4uf0}J`@V|8G?iK^8s?T*9X}fq>*p~I*3YpuHZLX z3``x1H9MUF7qUJAqaa-c<8@5*Kb)E5$qJyTPqZF_Hv45rQN&+hJaR%By()V3xxw?g zP0;CjlGn*40OPVeTk%LZTQ$wcs{;eFF(A>2M%zHMwgT+q05p?H?0VN?q6!fMlGmUN ztp&i@xFiJmI}dtT;c49)_cCnTAbc>D3j&spq;;L=B0B4V+Lr=h)3pIMMC=-Pn}Ts} zP%J&)a2FO>MQI^7gp~oda^^GJH-)NOdcW>2MT1ckqAtiXkq=xIyI~}Su9TNqYyM%z zl~=8x)E#Jq0t;x6J^xvD!L7z~|Mz@hr=dT8Mp8<2r`@Lt$+)fOzR$jNeY`^@6+}gb zEiWIkP*=;k@23c7=3Vx`mFy@N!@~lBGRNuwCoN9=RJZtz^e*VrR@ux#(nlA|Mg<5= z5V-y2s|_fA0Cune7Z=atmi+B&mg+3`bJvH9Qa1GsMTXw0zXf)0S5=4UgSPa5F5!TEB*xLg4J#bIj?NQR%iq#8YtF#f)Jb5WsBW$#+!0x+{AgMS9o zCK))pY4=~8&lK%wxkpKP&i|f^#9_G=DtsP0kZ~qyx@6jzh%+nJHbx3x6T`B z!cEOH*t66}JRX+yB+?6!+hhMcrYk%sJCi)wuodiVT)jxOfBf3^xw@8UR{D1~C3>J8 zC<9Mbvd?0Q6fozI-w}P}K6n>Z4n1HIV*laAy^@fI&F){Tes`MCO8i4Sw+H5k$$xVV zQSZegdwAdAYW%N<3i_|)Yh}+YSkia3RGtwJS~VZtZ+r5gh22>-7L`u2(-s}qsiQ94 z?{Zv$i8ud;B(GdY=hFH4?waO;pZDoWuVEGwyzVer)*z3nAjl^o!Od-iDG>d?KCOWt zKiCFKe1we1&qQ-Ys6X3B;3konK1e!Cm^&?1~!zWGNwKLrFEfDn?XYV z7^AN9_8Qc{xdo}ozZk^5%>=(~f@7=AEv90;xee0LSXekT8et$AgjdPy= zYWvRVVHnrGZQ#A=8-N4I53a!Fmz7K>)}4G2jP#ppkE-VS#Q#aMF`(mg7T6CoWBD~X zncf^V(njCcxEUcwuSge7DEaNtdY$sdk{7V|@Lx2Mhy>y^+$?$Pse767gMWl-}1O+YRJ{EPs0Ob(nxIovH9`hlc=GaLDs=8&hzwUEtXtsYv3dAoUF7n z6|)cqSphr1Yxwt6NYQugR7`A09<1-z==g;ly~04s!Ndit@tmyI&_lf-o+uEEGRT+u z(I!YT%NhcDAA#xk&*C4OkijEBW21R$V-|kGM1?mX1>N0G7Xhp8tMzZai-9u_I%eH044dua(kbY;I21EnPXE1W}d@xumD&Ca;|ztX&=2?V3F%kK`5BorJ@;DdsCh z7&z4D!9ETr+SXX+uSl&5yK?XY?sP#n*H9piZkTa~X0N}jzc}vrhqJBWybA*oF-UMsmzVcXp79*gKB^PiVDGuGDX?Ma_x)HE5|NvNeS1Gc4_a>U z?vDPAmU!{uITs;&^i5(k@*uh-hUPJh~XtIbH{wHr>Su`mN~4+r8-A^Lv9)j?mcYq$xBZlOX7n zzayS$jn+DGQ0(3l1&+=uA>Zh#mwUjEO2GVi&VO?A?$nbDoreBEq8QD19e;4>JI@O19$F6wG+ zWLZgdy&YoUh=fk`vITZF(O+M3=e5bt`8O+NFA$4IjivK%@H*oJK=7Ye@;9>_(1m5M(#8k#bmHju zCk{0yCDnD{OZSeWLPy{1Qh1+*MHZBrA6hESV0Y8^lpa-bg@A=cQ@gkgNROL+nFqU% znqN>y@|-NZzj3kPA>`}KoqO}I^o5~h9-IMnO+?1oZAxP45SM(c24(~N%x{O=(}!iR zP}H%n;hWJu9qgX+j@*B+J1g5bv)i|54A;(IGPWIU@9*_A9@~F>-HqwboyVa&GxGmQt;9)y1E;zYvr>!-7R#tpJ!LeX7Ou*vIj@i8TW&EiNtz*sYkccNI zEWFk6-hiRq-2CKf+|S5o0Che%vN|bVUBK6VBAR7=Y7NxG*4qoBOx74`z(;+>_joy^ zL6cs#8FLbfarrvTxKWDF@`y;L8zyt3mST}w&%9>`hmno9ZA@Qha691Ag-e}6zPhg%9K3bQo_tuii z1s*=bH+i4VeBrCCEXm1d>_k%UQpu*!P?5y>UH0RX!b$6Eo5j<^rLXu!kQYK+Ti2voUxy}=&$LE;y6Bbp z+$A9*!ALLo*o=PburWR)AwKo1gF4b#|L0Vc^b^yd2ms2MN}XNmC*H6dM4VE3)-hkl z=!doa*(Pr>Tll9l^(iA7vry99sb$XlJ2X z2%LmOg8vOiH)a{WACx-DeBl&*R)hDv=t)*BZq40c`twdI%=nKLK_cEL$Dq2@u6>G= zTCDjkr-z{fWhI-1#9HRX%kNkY9dZV{Ba6)+W3@irW8NL&W|V+BIj)TE>Ef~MSr@IN z+sHxdFneNidMyFI!8(S}LgN6XtX-|Kp3!#ae9Nsgw3ldY20Sp}L?Y(;0RxHi=z`Wk zsc!ad)7lgS#QST2lARWuYJYqjnI|AGFT>Agd!eEdR`S&iviCF0K{M7A^iUYZ(bcjk zOUowEY8*Xcfa38$mxuOZh@xoVHBs@*F~iGf2f@ zLfZ!QRcX!E-y}-Z5e@1^|$7z>pDe<>}-RyNA;RB`Gf>C z+f8!lY)FNxb9e%^#xWeChuNPFum3Ri`~$^ixHbv6d=Pu&FSYqE+5GzRsLRG=&cgGh znp1Hn;B^0B-L=yjE$mXJ!zC2b{!pSGn8P{~c+8Z4nS5g^)Sf78ZuYPgaKkC$$YRHw ztr%3F`9)lP5O=o0%eV90K@(P6|3l#(^WItq?_^;4_6|U4H-kJ*CEXfXfYhwQ83iuw1{6ozHjKvB5J z&MV7 z_7FIre_1XtC>*om%vfdN!aZ}{ZazpLPM1y|K40~ZLr;m`qV$ zZ8-mD=gVU>+WU3~#DnDgC1HK122RQ4b=^5p{s|y8HHyXnWKadyLo>Hg?5*i&mi8y9 zzd9-T-=~(0evwLL!|xSq#oKsYO+NO?B!Xaueks+9QOFvU&!9~LHtO)(&ik_u>Prb+ zZ*`9-R5$xRB3d11q#)c#bpu>(;e2Ul-viDc6gKk1WKd0T?`3ygal`cYi^0b#pCgE` zbfPrpGF6YWUw|d>6B0yrhhpIO;nT>wEWfC6%?zT7UV5&~BPhn1p}89#((~LB55L-R zFieZH5X3 z>eUu-*F6Z^wzdKj`oIy(BK4gSxe{InhD%(=GqjaA_SRMmEwgzZGUk zW)9+rmM8a%(ZB6!yqXpskk+-q&MXYKDZxOhk_y#M}jd2rp7F|W{^ZI{_Uy&+m zKPGvC=38_iWbEHbX*~Snpm!xb@kt!57r*~91|kkmbiODbvl4lc!7KlU9@h79b@z-; zrV3la?t9&My*^&qSwQ-%`zecMP~hc*YZ&*y%#$*X6XP4z@hTk!NzLj}C2RseKDb;W1W-z0B+v$6G~&C60$$;nbwqE{PJ78c8J&Il_jYzxpVMsfCqlP=9@aQ zuC@7!8z*4$yi`o`92Qsi8@U^GH8Caeo^u}oU?$kwrt|M-G@SSz+RnKSKfLkqTP;|? ztnwbZ{>gH`6^*@snw>l74+GT8OH`{T;_j`EVLVf2MJ^B8|OdJP^!#Nr>Kp8I6#@Wt}fJc-z_3j_3? z+EB18_XH~4qHq7PN8)^}DuUpewe$&e_>W106H${ogTZgX$xM?YlYb(SO;;Z%cb4u{ z){)@hP%ggX7UyqSm*!)^T@n6UuKPObu<<0YWX0Fa{6iI??q#!RcUa9gpTaQ7b!2}D zm~_8^^eoe%_J=o@R-q`q=|gbH=PQk+6?C+JhCq<*mzS5fx>PaHHCF4Z1B0nw(+rzW z^CU0>@c9ZczCN*i{gJPAdwPNPbUT^7_vfgFJP;`KoobOrZz8-vot3W6Kdow~jZ;~pB1=jVRQv=eU^oR{%5I5G@lUgh>`CZGC3 zAW*Q==x9iXXHsdvwRXw6XRy6BvQHpIYj8I25x_tG6-A@ZzJolaMwU!+1IOW>HlH%^lZfw$0&rTkN z&-q73;t_EL9=P;SkXv2*p8IS)GqzZzZl=tjV<@&PtW16u9yRyE3Q;l(%f=Zp7CIX< zF`2y(ObEG9lG@e3)fyReK9=WTsgLyji!eX8pgeIBG<#WPK+!G|$(PotkbG($_wH+| zWmo!%$vOc}A9Ap0fqp_G$gQBWu`4!#AR&%k`0?Lrr%^=IhVE#qYR}xMIh%AHdf0W2 z`d-lX88Iw{5=Zf{f{sglsW$d-<8SYIeOrbJwysu1;z%(@6n{hnbC2$0(HnSHviLZ$ zl-OODqgj$s$@>1HT#;Rd$B2EL(=1pVfjfF?-i=8P7}?v6ANzHnou|)bIv}N)THSJ z>>E{(>wngEh7Gu{Jc)NxP^emvSekS^IbdHCznif3@Z)6R$yXoO;;Ws6zY;9rlXi-u z=<<6<@o8;O3=WJh@c7;Q=&VQgEvh1)oNt@qv{u-?HOZj=>}!<{R`s#ns=m6^3Op?cxw7zRgTv#S%=8jm5557w6=( z?`5STeUkexQbq$d*4H~Z3LLP+mN_LQeeydePf1rcdN(xK5+E>+LurR==|)*0Nv4mF zk0F#+E7URJG`Tj~d=R47nud%F9UQ{+UnmR4e zb#yBdG;FL~0r?`Ubs@@f-09t;K({naSS>`vI7K;!24vk^s~$@HA6Iz6;*ouD{# zekE@#M=4Zdr!+80=zfoqs63x}+Wn=; z1EX!z6Ln;-CQ>K6=4z%JazI%ue7nH8XI6eJLH?jCuD(V@xUH?Dexq4lR^=z4il6l5 zQ>?YtlW)h#nv*>rQl!x_5HH%E!^w0^J12$>1X8*Edx865`0lJJ1?zh+>H_VF<@)PT z^?y`TN|TZr2IN4v={vr?oAP}%EiEl_SPh&+Rm@r$*ylNEsAaG-5McE@$kNKH6U+6F z_aMgrh;`DV>1`~I1|gT6dpZa(s_G-_ zP+x7o6iOYkoncxliALE@ua1;h>7MYro!3xzNgYgs^q#6M&@ZdzX|q0iz;bakUgOM7 z1AW8p7T4TyMI(BMWP1Ym5DGp;kRpap;y6Ky;?cE6y;49PFFuSqwipEj-ZkYwB(K%D z4IWp_;-nnE)A~2rSs|bva#m!9@R8nQWvo~vNC=H_R7hdc8_LsJXwlKpF^;u`KXRK( zB(Wj1L}7bL8P!z|=929g0P^&$#g`M1`&MZ<1MKm7tR%v_iuo*mfPfoytQt&cWR4DV z3_?xXp1^mhx}W7Pje+^E9}Cy?+#=nG_$ZdanZe0Sd#4I~2H4a?7$Hne0v@{Ct9)%S zP>-6|iv2?&#f0}gj-fGtK^qTH+3T3EFI3~WBi|>);nlNLf4)*b&VLFF;*Bh#dnzLS z+n@@p;6VX=d%nVpktzpso0UNhif^B(XIw#n#yK!4sr~(N6$knjx68i8P*e&tGx<=4 zY1R>2#JX_F+m82M6TXt4kK;%OBGoE~(UJ&ivJa)&rE{Agwy9cmtiNpUc4$P&ejeuK z3+G~F;R_GxlY1|5lsi<;OXE8$_okN#+WYdA@EJ1XW9J94R9|Hv{&0)K#EX;fAv?$$1M017 zg$87G5yYBo9h1S$CqIDhpu4&A^Ph)aD52MXx5h)|E?%*HNzrz+MO?{SEmq`WKeo=s zmCz}(uti)=!FfLBgv@3+gxFx`WBM}Z7J37RV}-B1s6&j8RJjy7R*p}Nf0omxe2>Pz zHk z2xHRqQmWio`RJm2O965*Ek?0S^dpVs$|%-+y5MvuKT^bDx&O=kjCc(wo4Lis#pSxE zk?KtRXYI1Z9#h?qGu_=G`CR)p(3*Tj)Pr!w?IR3k)-r?kguUCFseJSmm7BGn`~#nh zEOXnuBF+CQ!3!+kfRLrB>B5%~ozs|)N+ifbLmL3|A^^A;nX}Ba@KYA9^tU+)=>3uY z5QW|8HUKxj3n-;FFWps6%F=bobboIv1-Xseny4?k;`4yH#RN8rv$50=D^+R@v1$-= z{9aL!%hE+%Qo5s&H8F$a1L#U2D(Uxx)hV>4XU7a@!qmkUDgnc zNEYF{cp>Mfr-<;f6vJ$Vqe0Bs-nL%(=mp>b`dm@<==tQrilW4UgZ&sR*SLwI_*){v zFn1o~Eo4$U^~I*INJZ2pgspGN%DgI6o~mLON67Br;6O|m7$Jl6V5R;Erw#ragznJ0 zG?wn%n1bveYT``X6#N6#|7TBw9zJ~Kol@oXb8{#iMH#^>vw1h|UcPy0Z!YB`gl{8- zUQ>J@JfN4ao&5EFb4qOJ+q+6gSW82_Cu455(MyR)owpv)$U?nXFsHi*z0&oHC=U23 zQ8YU)i;4dQnkv%k*hzIVfp9-?o=H}T)@PO~k7rMq6^hD{gtU3V`F2o~K(a=~>K3|2w zY%3HJV{C+RVTyF)DoX9-!pDKK3N)kZ%7K=;B(Uu3h|6~c{Z<&P|HkFt`0xL~P*-sj zVJ`&H$ED_N8M^3gcCF$cmI`C||MSZ*=F49P?erntD%Yd;QO8hmYzWqy2C!D^DXNHr zsDVPv+3{3wn(VZ)moNxZ=ESa%&wvU324Fk4JJ}|X*RXK_`MXw?Jk5V~EG9>#s5*w; z&k<6E;0rtkt2Pl7P4Wu*E}$Rx!D$TkLP>EM{AMw6@un|^@wjSWr6L3^%sWuV{B-~Q z#g!Ez$Mtba9$=@~ODUxV?>8tmVIFj{8$g-gWZ-31RaR*=QDrzqwh_x~ zUB%4p`^MsYfrsgqBIgM4IfW2fPLpNHLb(f|(<{=h@&$5o=p!FT-xNYzSfs8B*5qQycjgp zzst-|Sjxd;?Q1M%1A4tS?0S)|Q8La-9U6c$;wQV)L&T4d3(g4lbgLaZ@IwZE7oe&- z!4CYB=xI)6&S=5TLRZ(CLY6^^+L$VL1Ll@?JM+8THB^0Cx2(jP!I3H^!g|i2HR*oN z2C&3TYralKIAABhR2T7R25#bUUQPJQ9KyYEyu!9A4Ls+Z_>y5j@~m(fGPpP%vS~A{ zB3YTG#?LFh()I5XPb|&8NX5ITul{mV0hDtYdpr0;r*kP@xD{#0hv7!R(wx8Ac62+g z%?{k*>y=jTRtGft_~vfTweN=^476Xh zn+xYze{^xt(~mf2PoT1=%gZhOmS+vB!nLv8x;iAU!irf9sUz`=5S1}&OG{$6iAeRg zO6qi@bp~|jyNi3TcX+}->WXHqYSYNR?TDERK7zX)$@1Y37_sYzcz2^))=hg(d@(DS zwc;@S3tT%yF|;Yqk+0H*kTEj4WVBTP{~6BIIr>H#)D-{S0$!O|T!Qu)?{ z1V}xq)-v<<$k=0Cgcge8ZAbZHj%z*T;Nlnc6!D{~4_nFS1%T$_Xvv3&n*-f#hi=;r zkgasM+ozyMm64Rh0FrH0xvc*P9q6>1p$A4k00!oGX>jn(D3Y=K7?MBW5RsFWwWU0a zCMNlec^;WIR!A(|@*g7zwGya5{HU;JN8MO_AeQ5^L;rN-{!Iwq! z7MQfi_x1qi??jO9J)tr-HfDGB#TvvV57#)CP>1vi>+Nq$ZuBx=jJz1zBdAVTs(gVU z4@0p=hKf}Cnj4(gVZrLyZ}dr9=iZw95$nbNjdxwQmJ9w=sbnH!2qjtw>R=&G|S!w7rPQ*lo*~`1k;yx@Ml8+uhlzk2-h?K z8RAiqUi#at8$COz^M|X^q57JfAge4b!R@?K7DqKWrY`MBCs4BfsKZ*f%BYzz>Uj5w zhJr%KNujg{H+l*}8xeqB&plGcz@3tWv)^f2$ZNfhXVb$)#+-(xVybHoIlp{;*SkAqybi>kSoSxdfv?%@evsLS}Nc~;{ znJN-y9}rp3^iH&GGSr* zcUbwb0B-b}IM)PxlxkqSZz^iYellqr{&-+tvBWNUeigVr<~=_**YF^;-T0Q~-Z?py zyENp)ux1w}+78U@;`M7C%1^Qu#yG~4G6qrSIt!wbaA3s{k4iV&MI#A;cd=)1fuNBs z`>wQeL}dKX2`?@!aBq%4}qmay&k?@fwzbkCu2J z2(B7@ksY9W6GrqP`dyN($tyz39`|A z>895&@YwOK5%nb{+~4k0Og_uh1u-kD+sS*_ZyhS}_T!^(`A-BwQg%|%$49$6p}Q8B z=U3o{14~Xxlon?4vl2L&b=FRSR3&RMF^RGKwG7?e83lOxZm8TC28abs{7&Ja8z|*P zA}zpE`DVj@4RXQ-Pb586UGsmAGJm7?PU}Q|5*+ssMb{g_Gzl{ewKg6;%-x9j3)B9z z;vICiu~+;P(ocPMex6Ps2l``0zaq$6C2X~_PSM2Jm1!OC^w%OLeT}>e7e051T#Tnl zo`te1X2RYKp47KBpyXM9W@zV8KePCWmWWWN9?}j_tjt+~VV@^WB?WJRQ}BAF;zB>F zZ8_y>$3eS7z2&XFv+UEArV_0A(EPkseCizz*>?#TB*My>rPTQ?^c-hCg3g$~_1{j^ zIQOL~eH=-l&`G8c|M*Mw+E_k)Ae*Hsti-bKnLPn-Rkf8M0xYE)5t#`P?yJI%C*Ywla-;yQ|R6z)S}{3@#w3NiW+_Q};fX zx%_v={cX$3qsntQ<(~4BVEw1rW!@+U5V_1y^wyN+s0R-YOwqqn@nl_gg~6HsK_i%1mB++T=_(wAXVXrqR~<`P{({{7hN9qMC>*xMzg zhhO?v$K4)3X3R&Kr zjzFdtuPxy{g+BE6pV`!h`g}JHqnGCV`WqACJiU*#^>K3EKO@VTmsS&3XyCwaRXkIT zRmOR-*BUWM%He*BT0;IaK;m{Nu5q#hj*bCD_Y*$`Hm^J(8rW3%BK!I5L(6*H@ii&* z%a)?;kUs`$Dt(?2N1Z|1;)*tKOOgH6M}(bB=*2he287|Pg|QNdXKS;}Xm{0@cICFk ztGrf}^))=g9bj1Us_@Y9f`IlQAx1kPU!W7%4+vB;UODW5w4+#EMMPD+c7h@o?-E~c zX92CjZWUWA;=197kx4iZ8IoUwAZL5;`j%Cb8dDG9xuc#FgKD-YN|PSF`;TmUbD74n z>{C@ARNx|hJ}U9f=@nQYKB_25mC*h3deV_x0#cZUEl_)@@EkkP_M8TPEB?dGb>8?# zhgyChOcfL_h3|Tj!Ob-&M)BFCa2zHJ4&xfWUw!)Zx5mScTXG>t=(WOd5w;X|Banj&NaFrY8p$C9V z!LfLlOZ^EX237H7M>B;0yDMq%WnFy(+~hBm`m~Q{^xz$BWLE|pS7nb&G1-c=_I0^S zI-L|SL-wLkKg>laa&L`tNTF)+JPXtRD62UD7?(a7dk^sqVDgr9lvHwhkPX=1P)cxo zwLSY-c~pV_Q$#DuLk<4}=%o(WbkY)TW@imOk7tO28>}lw=-CSs&A0S$f5ulRx0+b) zDte02_b(}{!K1k8$J*uf3o zJ>FDpJnx$CGDheJJx;gXi%{(QBn}`ppyGVH4n!yGUN~po+|`Jb^JoXvITI33Yh)=U zA6SgIhHm7b;M$uiV*H3WDh(G!oPKbUt~5zTJEiZLWodh>b1Q|$V6mp0X4cm9{c;xJ zG4^{$tz|aT&v`kI1%XT*V%KrZD*6H74TeKfb5AAEFw?UyD6k1uZub=}P2O0sRyk3j z-y3x+b(60Tk;8X|!qN=E^1XflGEC;Svrbd@@+i*)3-6cgR|6B4A+)d@9)dX%rYI@@ z{0z~YZ9^O*THm${U?Z?|9^&Z;TdY#wx9w^hEXbW}MTJ72j}xD(xFRDtvS;C;E2Jp3 z9grA+Oh32xrz1G9&_JNMg7J*2%PT7c(5r0-v+=t)et{ely%8QBo=+-Q&8@ASc5-i| zi0kN#iMqMDRqQIpShv3zsq6CrR&v$|q?ti!DzJJsVy$EaIf-KuHV(CQbnMy> z_V;~McKV71p1HYEQBhM*)Bp#>0TjNhI~yDyPgkyMuSZ>KwX2B@?g;7K8aY1|Yb~&Q zvomAw;LseIk{?@&9<}GVo3>-QW2wlGmw9vF>H2@I)k~cb<&I1CLn8s`jN^|L+z8{cVi5GThH1O((e)8}T z_~O1eon&KU`dzJ6ORs|9YboPcv>HbXs31&H&Ej)_+xA+6PPzk@v9*(EtY2lHMXlkS zz;DNd6~xzRNsag#feq%!a{GudHpiMd2mzV)J@#W>V9K0n=^A9cGbu&qWeBCNsUW-@ zH~>ZK>9cxP4NH~wLuVYWZRpY{&5|JV319=CVBnrTRUm=gCqB5YdN;q-evl^TxbYQX zFqt?G7WlO*x6aQoxSB9UJm|OG-t-Jv!4X9qxxKy=4t_3XWyHT<0@i@f3*_`-!s5$cpX}ir$FxQ&Gc7sRIgqjrYga?Z+q(hlFc2Xj4WpVM*z6|!=Qg9 zzP$y*S1irZD7XsrN4qn%2(Pk~HM45w1TY~{`Th)4=S?jCy}F+><~u%X=3q>%lAgAu#PbvAMiS*4 z)YI+m?GY2+WZle4Tm)%6O)Oq%_;Nsn^a9=3DFCX($G@RgZ3{@*(*++!*8x$-6Jb91 z!>W4NwH;`$dRnPw=H^5YRk~kv+2%e>pcon|pb^20$6bJ)#wQgIa0RcstKZch{7-hi z_MxG550)6yeTCCK`5VAGr}eT>{^Xm8TOIYr(vttg0eHf7VXCBlr$F(U zsQ49Xy|l4Guk6KRz#&T;5Ob)OsM?xeoTDn=?dsg#7$Zn-MeSWQ|IYyF`@y%-PaK1q zM{)XLdHVh8IIu*a|K}m40-|hsj0NG8|M701@|VY)-6@IFvhdo!Eo9dMbFQ0>smTAX z(2i;Dts^^C`K}&*@y;C~TQ>(Z?rBZO5aE4Io8^8M-h5-XmAvwCKGYUWv-QdT@R7Q)>EfKwyDNlQw-(`hA}dI%!#>mOeSEu zg)=(W@^8CN@5kg>q}{xn4cK_bpN#@&e`kj}=**2YS!UhRn)Xqn7tgXZ?g za^wT2rPe@WC|?Q&Yx9CyMTqk$=c$!P&2wUfRFZs)6cDPk)lPf&4-u5Le_YP5zCw`r42rV@2$6%!jf!N0FXDb>zeWo7TzTPY{N6v2S!P-KHVG zrfoUKr>%e$7@$W1K`3Wrl=9i_Q{Wh_g!0#|s@&>!$F6q3y-{M>lHf#yz+mOm+QN=K z<%@xLLc>e_sCO%yDZ2>s#^H1{c-~xbSZVP)u+<|1^2n``k#M)d z*+TOlnjL1eVZkH&Sq`PSZHm|c99V?OK|3@A#uF6(s0B0%P&j7Y^=nEfoacW&%o01* z`=^|I8yqUzE;<{r3smUR8XW{-H1C~zw^1ey$1g&R$NHQB_*?p@@+hzEmJ9m=%;ENH9ta0A0l{4dXW~iK?}z1j&Cww zPY;b+$Yw@p|J=`&OjJ%ly~%b=z8YjYP486tHU-6!Eor3u$KqpYOY(l+s1GvFyge!` z;>9?YG;W^WyjP&DTF=yWL8dikqpAbUd{ZSVvF^OfGZJiZ9r8(Vy3l2k=fzY?bUl)! z7mN}`OnsEgaNAHr9`gziRnrfDu{Y?p3!QI0i-miShgnuGmi)7Abr6}#u!J$&*_jw( zPv}{QQSal_wNs7eqp^C!d9(N4#0fcn`9iIMv+Jdyo$Yc%ezpMuT5KjJgv% zk&bTNGvWU>gKA127zQl^zqo?YsH)D;&Ir_0WDe6$AZhsYcPf3Q7<;@7Ijpz;^!7eo z)Xlb5l@AeYO{V&ly~L_b4pXF5$LqCz#Ndx|(as?{(zNV_OdR%n<5dpr1Ajqck3G8B z?c&1Q3bfVQu+J0E|M=!0f|a%C2C~JK<+Y#-`F&UtMk*@SnIJ+RoTkEELa02F80Ga* zD=l4{?B}mhT~pmK%Q+sf<=U{v$7uHv$&LU4Y5w~ga?oFy*4$cc{`+tFU3$ve%>*%P zya4?p@F&p^D67OCtVa1j7&62700tB?z+1$;u#}+nxb4j*l4>6R^#ZWt&-6vtJgCwk3=>EphjwX z-tZVd%c8rz6pS|hMM-CpzpJZDkwB2Pb&zDgf(k@x8Td1@=)3r~?P_e>z&LAbmW0l1 zDVnl)vcERAfJ$DROF}uz#Xjp8aLjvSNuo#XD4&^2Ml;l|A(E+ajVM{u(1CApR6S7* zuD|zQ8*K+%pOrAHCt`2%c7EJdsMrGN)++66q#*rhltIoYJrBvfJZLuk;6UOQX^Q{4 zN|l>T-2xu|V(#Q3`Za7^&Q?gh zjH&LE)tWO0gY@gCnt72j-H4vPKJLuOLeGzbyEX`AOG^CR0sBPHpC60zVELXuy$7|+ z(yg&^6--t^odE_p0#=0-^Dy^mhxy->QS&EG8V6|nMSHjQA$<2PIBIebc^&U%WRR5re0$&+|K8Sa2_eNk@G_-78+hA_jdcLUK$^ zmf7)CYyUcQ*D}*UT8`3ALRv2oy;3)Gg3*Zz0N9;uRIB1-yt5Qkg0-XmvM^IvhL(~p zr}Q2o*gr?4!HZmh;)$b>6Zaru{XiCBlp9%y>zr(8QRFcUDrq0%CMldZ`0Qj7>aZbH zA9DNW!|6LmT7BiA{X?;ch}?SWdUc0!E*dvD%spupxdY5Wpwo?OSH+uIW;D(%6-vqd zwxTpVlJ4`I8}l<=dUG@SeHu-RFB%80bMUiGeV!-ikXG~)Xn-ap4J9`F%f3<=M~+>N zOZ~oO->Mfg;vagcu|7w#8yW^Pgo);bXq2#LBJ^1m=^%AM0v`Jt5C*7&$y)SR7jw?| znyq+=@cUOyW&nA8N;yGajss*U)&5lGD*V=kT%<%6>{Vy}Le~x1kHaBtfvo}N`YAOYbRVCb)Xl6I4Y*Pkjd1(SS}m{?yh2&0)QwU2^_$F!0$z z!ATu5wAT^)4S6i!xEf7e>V~gEb`3ojq0xwdz6@B}rL3vG!o|Xu7A0fW@OY&_5kWFL z?AWeU-IL*}RZ>P+_~KxrKDxIKB#qE@vc&0oY8k4*2lW3HEk^6Nqt!7=?NM;2tTrC1 z;SxhDnp(M_y6CXfGXNyyr;&BLol}!@y*$@rDAn-I;B1I?<@+>QwN6|3t-bgC`8X*x ztX*v`w%7nj$+)^)Jh%$c2Wm>9+!bS@tWN{cF3b=pi`}G7YdrEubQ;&(va-3TfRFU{}`J;IpBo|v*;)E!;On6m^&>V zIA0TIjc_aK5`(@mcR|Qr5M4WbG!p>VrNTmn4G{g2PYK_%O`Qt8x@Y~S;j!gI&TGtN zdK?Bf26Vx#*B!vJ*OBV#kNXOeI%n%Z%SK4U#To?4H-MA+Rm)cT&`fC|H-7x_e*3H# z&624M{mAB8#r}XNb4p&5s#a9AvG||W%0?5`oIv4YRTT%H1kG06c>Ca8BL(=42<qJ-AQ_&qzj8Zd*y4l!x$~n0(`MH2X^{oQ~XGTqU^03;3-( z1rFlTkn|TP<|_ITrq#%8XHdE69W5S47|iJ$9mL9awz@UsT58dqHF)L+rYW7e^~Kmn}A{wKNlo{E#$nK;Sm1Sxa zvTs9{u}A9QoN?^CA*8W}>Qm?x4}shK1MMSSCG=x4 z*tP&iWn~R}>;7lXpnH~DTVutw>YC(wvm8bcF0D5k3v>;>Aa*@>o6~)p-;B->Ppykq zm6XWk&z|@Hbra;QXgTN49@U+ZUtcS@yEQ(oohu>sy~H+L%H)ZkZb+wR87J2+5W%Y*YmN%~R~tYYKl?OvY#yuR-f z6+6lPNqbiM-F01{Dq^510OKM0I>h;Kesd#xKYC(f;(%MEX`>%WJdx+8j!Il~jDx18 z<|JJlt3avxsw7t&Uh22{&02V}-GC$-b+fUWYS_#(IaY9}!{C0r#+PNsg+=kCQO~I5 z1R0E0168cuZ^};p4|ndM)|w4YWY1HyC3DoHmGAvHUJD>Xmfz-gbmIBNlcr*UUM3^& zo)I3KU6;7*D3tu9^UwhunZ3qQ-5Zk0B;VlYHr+?{I7et#2%S= zRX?g}ap4_*82@}ewGGGJSBMW2Cpz~OkTv-m(#gq|Id?{urw7|T4{5EHOdK_4IV(Af z!X?lTqB?%Tt#2b4Y=|;TNWa2+)(czKvy3$Tz~AuDbJXrUgtfQ7{Eb55NAl<-r^m;I zGm1~4Y`&3A&*b&XyaJi@j7!dX$9(Wo@SD8R+oX2?N{Md|*6`KqsoEnMsXytfCi(_w z-Jn00LMX3P-0n4za0eoT9@Ps#R@RG+?xH=Zcnz#nB>UTG`+I%4gnhN+zXtyfx{(M% z+I0#0Y~B}Ugbp<7!?@YS91)L%kjE@6ERRB4>*ZGI5J4GcIHPx_Viejmve%Qfb&B%L zN#yYCvd^vH&_<>ZCArBNJd(f#5%L0C>si90n6+X0}I%lB)Wt)efNP+|kRHgKs(~$uVLx)H71QFrn@nB#fT6 zB+=kHRJe!-hPU6y(4B?81|P$-yKY)yQa@?=o6AAzsWR8`;9gk_HlA&SR zBq3-&DJ)U#W?N$4GpZLz#4Lm6K|d@h!OL+r&1Zu)=C0Ewy0Z`DQxTrXBBf;$6Fv$O z-vJ53-M*hetn#2CPNk}?3K7MJ0iaPanDu-Oc*O?}$oz}Cq7F51>mu4;fdvB?mgclB zice`9`Sf8he`#yUGPL>pq5fzQLreZKXgDYtJ{9N&k}=6f-mm*CPV_yU0a)%+{>NTf z2_YcYH0Xgun|I5U-rfV>C?&A|B5zagaXC}i^Sxb*@&sU{AEMqnuSnm~X} zB%s2TaZh?+FGM8%PLsYLO>1dRkCdsd!A7p`I{#PevuQ_^x2^VDMRZ z0UsAdMoJCiTbMK0tM`6ns3tVm(iuqSYkxK|2SSDwXZo(RprPx+V+Q0s+b3cOGWj~m z0(t`3;@<%ZDsv$V0PxN!=<{{29A*UOZ;r9;V^MnkP+p-7lo$RwqmIE~T3TB6fOJ{u zCD{Mu*|rY~nnb>XXZC=C=XzxzB^(0|l>hrRIv=BiQHNt!Sq4g7lMl{LPO^i%ajcFh z@ipV#-8wAM5>90M7w{aRfatOzZN@_D_K(k7m3#czc<)<2WQS;A<6|FzxH^11;H-p@ zM4>}>ZvV2Q6}7^$Ba#}-!WIIu-Ld7jt~yT)ulS6Yq)oj%W-dv{T~BFM%zA{Kx^BHd zOt%xqn5REG4eDd5P{uR@{f|=xD`L_Eo1zC(76c^rtr2Tf*}VRWbx!qaD#}Y)ptigI zpuX{nXEKUsPQY2OA&fyfs#AMKVj#}LS)Denphh)sy8S-?R!+=YfjonRcr6(#2Ia70 zt6Vy|bh~?SP!%#~2@M(^&>x2be~52h5Hd9Bm66Eib{V6bWdkef_4Wqi7lqN1;L$r`q{BWm!)l-Aj3P1(2j zs>J2mc5JNMIX+avd};2U%RaPdLD2#!)71R0Q<`j69QpyTesqWGoblRKF8LyNplChS zO4oJF>`p+e!%uv>y*MNGgZaX{-U5RWx~493z@gh##Y9kRrR_X4<0^NYuvU!irB}Ul z>pTuy!NT>(J5BTiqU6%k6+Pj?VU% zwWqqC)%9sr1ew-qwFZ85D%O1Ji#K#k$aYL|TwYo#^S5tNk)anht8f5Ca7fZh1ysY{ zjeqvd(d?4^Dq&%D>QB$>BmUF}aY%_36%=Cn5?HhPeO;+<32}W1I47JBG2Q)0pS3~b z)dlFOSnAfI2s;B<(Ngoi@j{fv{M&UipVhIe-RBJ)cTn}mjMnr8WzK;}eYjr<3B}5_ z5-5xbUOw4(sh^ZBwMmL^$|N^56?S!Qc=$mM#of}gC{zsa#){sUuJ^lYp}VWM9^ZOSbsJ?a7eZeSOFfB4^NE%AZ;-hPdxexs1gXG*m`?y#^~=t} zD(x~ked5l3>1b%_SX$KYvFmnoynxXOZ1C{y1pls7_GOVjN`Q_kiSIJ)U0z;hMLJ(M z{NVO8X>o>zOipl_sGir3NqkrXQ8{&fewpGt)}(O=eT5{2J`tlTrmJ+@Bpzy(T&PX1 zM$5r&DMBKm*sfvXWs&N%>F2;re>}g{tZ(p1XlyZ@=!mmg+oRU~b0Z&6KXHppvC3TI zBdW?<98BGAX1HB!n4~&Yj6R&XW_g&%XL8UPss@da+w9LoGGdQOaAAW>&kJ*DQTXMY zEagoCMDL-$C#@A9j{i|3|GvmD{7U|ir;&k=$R7fn$8gn8c%{5XFn)9QwMT>xV}sj@ zKBV@P6{|URfCSwN$5byot+_D&#sVF*|F3ZK{2TYRk?tmV0z#S^)h7krj?QHg^TsK_ z@^)|WREKWhs-ax&*iQd-#w^HZfy`bXphKw@jwJlx3E_fqxY{d7R?>d*7H@*vFm8

RfU=MIhvnTzpK9+vw&;*`>lBprE_dEBw04N$ z40}kzw&oK0r1x|mwJw=EhvZIg)fD0s)jWuZc1KTwvi@3{ZK0oLudBJr)6QF6i=}>8 z;guyQ0<%Kn=eQR;+=Jr%8E^q(?RW~ZvEWn3M4lN*(pGuMno>DPxq;J`v?vzuzWtmv zo6~GJ0F%2xaL5NukOEs&vhrmULw<`%>r3wPBd#_+b~w?+)s^+9W%VJ)hYnpOKZ}3J zb~qu4;V1hjGoeI3ZMtX?_x!$uC31P6#WZufeq#|!)$$lzScxhk3R5*3)~4#H`P}3u zo6lcH$PpD*ZZv2#P)_kxTegv#(>=wxX!E!tXE9Bpq5T1K%&3_;;rohReT*-c! zCE4n8Ij`4MT8OV@Zi=wXH7Ggsa+ZwOhi;s@)O(6T6C16>9S|G2k$eAG=k;P_X$h-g zNZ9)?ulJIQic0ClB_$6G+rVe_wK2PK7SXfsveq4OZHvV^IeE1M(B1DPGsqe@HmloK z%&Tc<)8J59QK3{434D;86_vWM^8EKv&4u(qSIS~sO;n-aKYiPXo zVx>)W^2+gnv2O~;oi6)s6wtI*6fnRww9H+Y#ok!pkl+IN3Fb&@>;jvB`=X%pi>~~o zvx7ZuqHHmRWTo##&-JV+17ZW?igq&aB zAoB?GR&)iKFaC#$;?%R*m64a{Ju#LLIMa6im_J&me%5&o#JL>x1+BcptLB(!r=E8; zj?H0y^b5D%=G75Z8*_6dyM;P{oOEA%spy#%hfjMpFMNMV6K7|D6`e)wEUqKuP@Lh9 z6Npl1!0B<9s%h(6OM1gCyT-{$zAy75T-4HB6u0imNf|>LpiK`9gKCV6kQ5wt6|B*j{n)QAO1T$l+M3a44#X?rexL3dY3AFOS`-0TeT4l`Q;$mry zxnWD;gHC?&*j0Da&qknGp)9{WmvyM)@x?`JE6K*jMyEAJ6|sG-r?QRj3oDCd4!RPP zE3wErlfLetJe5cBhEBqk3KXBN`6PNur!2P>8?NlUhhSU0Y6t(r;>>ld>V`r)&6haSVLq*4`Vaa~_jh&iY?RYGt0eNI=8_keiW+Ah`jrjv1yE<4o6{$Xog!C`mlr2v zC7w%_g{Rd|>rx(4tPchD%4_)ur=FV^!1urqp*wojcAS3(>xuf<87C!>nf8>ouCH}p zJ1gXs8xwi{rpP`m#jz6;exeDo?^~p3j_>`tf0nlV#=j#=UB)ufOG=MO9Iu{OC9Xv` z&%a)rEp?Rg<(i)rX?48h5Por1+guJCxZJ;wm?^47d6K;5-?k7mu)@#J4~12a$`uhV z&Q|}mW;Riwgn_Y`gzZl z>QiFgk=$K5C!G6f1tX7?Ok(Z}*m?{WTlbSPC-^-iZ~<*L)SNltH?!k|2|t?uXdMw> zu3W7@AlFRR4P72yh?L9fDL|8tPxS4;F>TY!h<)TK69~~fFjpH^HrrJ*c#^Bay1JVJ z0>iV^yZHiM(X%mhT+96ANllFINuSk;bgQgs8cCoi9Hxeo%Nz=JWNtjn5cZoYeYa0x zd^Kq#o@AL*i%VrydXZ4`a%+=#%s-kli^^4dJ&8%Js;dUOh`i@r=sO^LN@%0(T2{>4 z2^#KBG$rZc6SwXP+@~L74fcd40aT*zM7-_FCP47_E))IfUH(~zkj2GdU^bc3p-h9F z>2TiL9OGd5uJYfObgjgdpJYVN$OEn5m5Gzw{UA3t_sH6EgK!^PrLr!>B#9LBq!rr{ zWqtr}ZKbd_W;U{1`Ic}`Yu&cF`^%#@QxF!?vyQZzPWAWq4~@U)2c1YBjeQz7`wGEL z8q0G}FzX*v$%UJLWxG~)y?rsF=w z(>6*8t!^#8zG2<69H;v2IN+vERNRX~*P-aT^auf^{!;7HS6|DwqLpRFFD9gAnu!b| zN(qC;ILP$1Cpv{jq_0SY--#!1#Ub%prj(W^y!XoXP}zu!1{yk1;Cd$J z`S(?x1=*hEOzC=!wcf<>aP&g79D1>ctI>3y9rm$Aq`^3q5Mdz|3>1(|in#GLEh)|m9DnIF>Nx-2M(AYvkT){2_$X5@p@>&@~zO9u&s6YWatQkD0twOp>7FGM@Lbw{zr zyxm~UQIJKcd9E%bY#xf>bh;HNfBi5yYsp*tNgZIGhJW6XiY&Y1-2&@~^*bn2Rs* zW^%<$PEPjMtICcCT$f~*obT(!E?h zxuQN@mCCu9viWV1Sw|`Zl4Wyyq}r3eq1&KUCw}dVIVNz50~cr>Epc>1TbdHs_9VY_ zY4OWN%vZ6Y19blWV*IfgdB=NSghC@^FlCVLRh&@d*tgtf&3z&!wV8Zm;&qQ~8(DhV zukW<$)RREL7qQR9?S8@1Vsa>R&7_nV_gFVnm?5eEq-ZU!ucN!Wg1gf$B~P>Si{W8y zrj{Hb`t|sGzP>j$-T}xJ6h<&f;NAdl9umD zUtgO#K7B@%vHtx|tp*0b`x*#Sk!MwkQ)e0sPQ37H3U{oRN`^i0RcJ746n6i0WQt?Y zTt+`4XU+6J2Hg_woya`5j?JR#oaXYK@*d;Qw3Cfk&P15=y{lpYlbsKKH(RGy6l?;E6M5KS zMBSjtprV#D3$BVbyXj@`=~{VD7#k!x;19}W)d(|oQQ5VSAernUPV+tZQ`a$Kxy)tU zK=Nw{Li}Q2wcy&Fo@NEx)b$u0QY_5JuTTK$B zOR%GabfA$*Rl~-+b!Jt5W!f?r%94mp(&+7xj00XHV;V)xDM7fm;?I0$`W5i2us#Vz za2nt7iT=219te)fupMzZ;u48UmL7LNw&i5$pR!7O=T025H7FedFLHH$M{>U`H;^@WwAPB<_rF)7UfbQ7x; zMH{Lk+!lhVcfr9l;AMQqq^8N<3|Ee+@GeCxh~7>B**#ac>uqcMQWql>^MGs^isPR0 z1~Ywb^B-4yA*jJB%RSlMp*zaTkX#WV!hc8T8jrL|i}_d!S<5HC%DF*{bQNyhfzn(@ zI&rO=xRLhFnY8Ecos(G)x8!#I3@YfD?X-%zP+}+1JhvX!dw@|l5(AVv2W_L5j*N4e zV<=iv3Hp?RDmD7s{QTC&!T`2pe|{Vwb{5U6$DUF!{iiOnO_}P`1Nt`T>J+`rFj=W< zr>M_6@JUjc=Z9Qgr}0`=#$_g3+))El8O*;3lCF89gY zx})S3;7{N&^^&!(XswcuBDXZV_gDF8QJ$p~w0Xqw4g~Zq|K4m7-4wa%Orps~`wrj& z-r8|QKe4*4CXv*v$q+t8DbmN*9!R71>;02{P}k@G1%xF(flejf2H1!iXh9`#?II~7 zZAzcCa_zPJUc}T!cY?;HB&nzj5p9XUg;AaE^h83Z(cmjwmyt{_Bp>b-w<lQ#&pza@FzEP_x*?G&b_YUlL8lQ+3oiq9j zIv&>%ElQDp+M6dSr{{W4XYHR{If9E`0p|uX#d2&(P#e1%-Bdn3TgWBucVXXCV`-lba}PgOxveq(L;s;Gu)Gj{D~%B42iV&0LMsN5se+Ib}2 zS=~pBT1iNm8t9T3Esr^mOedewiLw6={QBP*jU8(GI(MD^2D<-Nek0Q1J)7+;sLN-% zX7(Y#1yjrFsxNb*@xw8mZK35WbR|j%VD8SL&HD&6ucy!MXI&leX6PjR01xtGek81; z^W@*EbIq+M0s%nPXJO#{jN{ZAK_|iZAMy`Fi2ct`dS@7e)#d`?ljq#}wVDFdfuII^ zsQ0}PNbAzE_rP*o|+lODDZictJ#ZrM;3FJoqyyX9f(`_HS4eW3Kw4IT`w-?l8 zPTRjDXF3ht82BIL6(+w`I@}X>h03O-vStD6cZH%kHc-qsQ zow(QFJADW70z(LLn$C*g`|p6xvH{*|7(RRfSZTLU!M7Lbagp8H3(@Xik+0ZrlYxK= zR;bz`TtlPu7z)+9eSu3M83Vjt(EnTq_t`{7nL1rExEskSud@STdQBMT!4QD$?;Ao| zPXw?6DAc9jp;h8Y)#%1BS0EoFzeCyt2mmR_XG-*7uq2H4aCGuRFbg=0a;@L<=YS@j z4_t{ba(MwpLnG-QW4lvTG=WGz1mY@+M4tMT3S`6PUcwz}Yh;2C`P+?z{dXV&QlcB?URLlf*hmL%toNL|-YtOG6{oTnfo6j~>prx3J2^>q zH{2RIk~y)jSfU!5lXyRMJ?r0FU{?d{Ezh-|`b*PbIdq3>u6AVmX6(`a?eCkB@80sv zz?1=c-~R80L#H!O-0CRzoU-ie?A*U=*RCCaiK|;!#Kom0+IlDr2CO-@0J#`zI+z4- ztvdqyhIDnAl>C(!uq1}B(|=|}Lu8X{hI>3A;=w5%7a;nF2qAFp%f*suB#;wW3D;n2 z^rd7O^%zYZUaA#jH<2Pj(gXB>-w}X;yY=|+@H!YzIF?1EMc6H~PNWeE8hLjb>&?OD zE%%Z6E8vFSA!!7~u#p%sNTzbXETV>Yt<_2e4A(875A8n({s`j1Y9wv+FeHe0W`?|! z9fV01*3AAN&<6J)p5Q&`4<_xxc_&J{BAD6LfFI;zPTW3y(B5R51m-AqN%QXpGDr6B z)WOM`7~S)L`)fE)iAT!%TN~mWE3!1wKFfbjCQCpR#)lWS1EM^v_&MM8`7%;-l3x_F zp)PCwGs>mE;H@n2qsP`Swn(MQx}enCk@*>C6BHqU`uV=gQ$W zH@$GA5I`e06G^J`Yi^j^5-D@KN0wew>>O+blkyehRpbx=Jc;d`ziZkeVAR7MH3j0p zc}bQ$6}+@mYx!Ed0fnlQeeY2kKrf#vWP6w_H8Jh;%c|uISzSyU(6yQg8f~L+_C((d zOcUPP_s{UD&4Xlhl8SM%k@_D^P1m~rn)&b{=wJsIR>5AWxUqB?$&q@Y?$T}Lecu)p z0V#0jIFl-ZEc2MD;05`c_4G-Lwbl6e0Is-xcf^2$wK{6#Nkg2MqY@g}wiB^mv{P?5bso?J&9j|c~7clY! z}RXY_{i}`oRLY9c!jA0_u(RXHK0|w)LLuWQc zJOnw^b#je^&?p4xkn$?EY1@D)-j3U}Z-9f#-s#zMO#lgjs{vn4f2kwpj?m8~;YDsB zuVyzPnp5kdB6v~K+%mu(Twu~sK6V@}d%WQ^u1ljK1ua`B&Vu5j{6py4Mwu_^sp$n@ zLGKC=d5&~v9pdE-cwBvl0DBpiZ^bZR${hg)Dj#)`7|%HOTKS$V^d@nuPEMj|B~|t< zU!;`y+Y>@wxiT4{u`z7b7iA>!95YYk=tO$aQK-w8IQ0@keoXE|J-d}NaB>3@y>@W$ zi0=#l9UB|$-q_QV%)j?Dqttv{bYYpyI0q_s2#2^ZE;gBiF<_9hD3lpg6d8xBR(GIM zhE;-bAD4FiZ^v&>H~-_BzWp~w?$!4HQ7D;E#*()6a}WXeyMO-le^^5N&!q3}ODKh!WqipWL7~ihiKKV0u{x-Fj;e2CHQF0kl zdxz`H`fj?Xh(toec-R@3YWTgdtV{u!Z`KB2%DDkcYHUPRz@bGSq28sC~%G`za797y4m4vHih<_NMo}E&xOuKU(NM zb`6i*8F2A5!1uiM`0((%Ly(uJXW6I=;zfWOq718*GhvC=0Z+7k2`BOe@A@~N>?jh& zEoqz(&+d}ugKUFVp=o>R*(XS)l&+iL6}rl$vjD=1a;jf(E_9+FT$hL423|_HILC1? zLenRF?alAYY$(#xf4)$g-*sL}9|88>eS~M|X)$D3GrQdSn5>-y*vI%kKn%obKG3n;I8sP8EWzFf*!KIn_7z{zv zX}JB4`EcgGsJw0(_(+KmRY5{5o$DrwxPgO>2mYAlCo z(-QRqvSvN8ufe)sE4Yb7b!MsEKKmmpD2ti&B6Y$B#n-rn4IsB^RvpN3Ab#OZ$jd6I zYTHxIaMBw{Hp~RfxIf2{jT7SBpdQ3udJl`8M%Pm48`|g^_cJ3klX(ASGkjgX`D=h< z^R<&f{jo2!8pm^;3G=8caR9o&X_M%Fj1T*7#iA#(o@N=iQZZO&l*5|L><1?c0Ern` za*|*n$qyYhevCP|$fo*3U0${W7RgzI&a2^PV1@>-OGsi;)p`FekMGTapzmc?llafQ zB^qFui@i>>t6Xf;mNf2tBokPHsKS`li2wO~imGe9&6ZgKs<1~LuLGMq8h`6; zn&ZW(^v=?pV^fqx2t(Zk`;5ste{wp3zTbhhl&?!OE5aYqH_lFT@$K2*p)mLQdN0D zEFS3^Lu3CO+h$2zppWB6OfO(Q1LzGS7B?`N)<8WL zT3)UwC)`sCd$`A}YWt(7k$6E%w0g`*>Zkc%*aK!LCC%aZU(7=hOWrUasssFun^n9K z+Y2B{?-U|ECY?5JcZco3aEH1fHRAlQ5zDLm!VHi%=Ms$u#v~~3*6IV- zHA@B(*1w)~vREDH==d52DsjcT0GWhrZp?1(##fCv2d>3Y#$ehB3EUV9+@1bepS%wD z&JLtSOm-?wcXLtKNr9>vY>rk3=*dbY88MBme^Oo7_&y7-;dGE@{|dyZFvxOMQp^${1O)Gd%5CQu4 z@`1wr1&tnO4P=NG_Jf%y>Art=!iiy)pV-9qQ~&Q57Dq#dHrYP6-b;GU@>K;ntJcM9 K7jnxp`SiNi$=AAX#KeB3wnrw)L?SChxu>NQA4yC>f&2XC zs|%f}Nd&*#+7;^lW1|aCmT0#}lz0et$Y$6*C?TXX$Ou1F_>1mPAtVQJB}79$y-cczr(byML76x*-A760zxr&**SHL%Wbo%If#JFgX;={Dtm{_g@?;={>@*r2? z^T8?+U*MCA+`7}|e@$)MWce#KZlje?V*i_?SSP4Q*9`1i9mH-UGCbcJEZ1#GCd(geksI)db0M3hPCvn4UzSG%>DA+3uh%2Aa1K+0cYL>3SHt`OyfeGlepRQgxwL zGvgOsN3%z9*8hgS_>~j|g>fOo9{Vw-_0u@K8Mh1dCiPgFE`QarS;N<{&u?0y6-ntl zinU>mkHjNQ6`_(NU6g2HnrXx?u1M+)aN07UzE$`L;W+|BuF3O<-;uuVPc`c}Wh`ML zX;CBI6h^oe*73vdj&mr6&aCk5LqZzF@8o2QkWFGQr0<>-dc2_j*!?a~RdNUII&cL}ak*wL8eObWeeU)(AL;%yBJxVefrQULt|jDouosk~ zCAyGvE%O>)UOwISAjcAmDy1Cf0KxgU=^r1%4IXx94P~_~2PQ*w@5}6mpslseEMI>i zZYb!DT91K*CedhntHH{_#sN`y-(8`mZmRKnJ-MI7xTD266@n8L)r}k6r*YTMS`57Cr?(V=(*?#oqFSka%9C0Ia{Xjhxrp7>3pr*Xz z?SKDRx}xjubH+%_1?Y8ov`J#!fiXIzBbha1M7>yT|<#Ax|-6gUTvPmD# zM7rLM*+1Emf`eZ5!ASW*pr4=_2&nVJ|2O>=S0-gX}`4*BotQCZW- zw%n{h)@r!zgw*vd<3};9>5$hAeafqRMh)icM0Fl4eD}unHOzPv+!xHZz3+$1i)2#2 zqP=w|_N{AH;8w_!J1f`o{5Py1_Xvyq*!%+hSN-=I2^(=|^!0@NANSFXeyovHxWmej zpe&*xH73R@TP3nW<1VkD@<72_6;n}Cfgp=0OF45*nJ3*wY(A>Z=rh{rhc(F%nw{4E zR;gCa*3LPpIo55XfO}2RGcx`eFBKLPhU8c1GZ+r&EtOSNI+X>K^_6|HZB>RdHxvXi zj^+1KqCZ8)k!hOOL@S&7k2#bQnwAev$!94uWm9qZM9XQ5U8!BPJy-O*_3- zvs$}aPC}|hj84HQS~jgqAx>6XdiZHXN=!<%vSY#q&ZOM8ptnVDG2gO~sdBBg6&|>J z+dJJwaY=L8a~a+fTxQ!v-YedVSmj!o=K6xOc1`bEI@zoH+~z%eTCNqtR#RJjKTqCH zKX{C@eoY_8=!V6ES8Qus4>_?ouaoC-@EQ!5<(jP-#TwM~9Y$J*Voo-;JU0H^H&@YW z`xe)`>HCY^7bOpmrEz3LrPI=5WX5HLVxGm^jU0;!8OHY_C@m5@bwL;3i%l# z77`o+k=kxjbAK~kz{W-2ddp8;)^YkfQW1MYkxm+EO@4G3wa~|0pM<3g`=eScH zMV-LT%8mr|M?Zg7uU3bg#hu^1`4jZEz4`6{Yo+x2vNoMgW)mx=h3{IT0%W9}-P=$7CcDgnk%ofZY zVAP<_zcyN)TAp8?yg7A~>7mO*cQ&Snu0F4R^8U=peU}^jY_mA!bGZ@T+@1M>#q7EC z1{od#&-#MA@W^Kc7g|^KV$(JF0zNmtz z;iy#U_hcu9WNf32HOB3#mjly**f|TAZxh~n-iK>Im$=XPYMOp?S zt2JIvwoE(i!#0pLAU4n?*&)$Av>zxi^Po`wP?zeSHQtBQ?$`df-Gcc*yDmkmxTf$F z7T`yW!*r8LO075_~k^eoU1sjS4XEK>jz)4ji{EbuR8SZ zcTp7aX_!rznL3bdXlVovJvDQ62%ffgcKouD&g)y%xmUGk;QEDfIeyVq>0oMvw5-5D z$kuZbu^m|jeHmSf{N3AS9QWX;;5eC9846z!yi#^o4l4dNC#MsFe%f#e`w&(EaUNB- zS3jJ7!p;e22y5xsWCbmv25b%nMwEs%^fkuieB%Ti?=5v?73UVWbUP>T*-Y)6CSiAA z2a`^+a+Xwtm8yXKsdDbD<&o$q|c5JPczXKplJ!%2HURB?0sLQ;JR)x zQN?p^xX5yzw=vFBt~<$4v2(OB;rhlk?SNxLjsL!z{F(1M)n1%MoCFmb|0`FbgQX4q zsaaCF?Rd9kMvtfN8nBMJXENos2k+w(bAE#EzjnGDzK6@ilCm4UjXp^OXyBe=#uKtUu!Km}Kb;71UV{MWTO zB0U1~<$X|w_?jS~{5eJze8T^|13&ONe|#b(BKyYL@GNyR%`2nfOml8+xLJ0Y%3VwPyDxT8O(4kEZtBl=nxNix+3<@Pm$>sHUL z64gz8<`|wr1H#JaQJ>P$C%&FhR3aeKc$|azx`>9A$y+2UI}PI#3h{0Hd(eky@7^@* zQpr^dOt?;Wx^HgqdFJL1yLT6MPfbs4zI687jIBKmAZAkCp+ZDL!N8{$_Col7UqIcE z%;s{Q%KXm(V2Uc1Ut$OH|FK2~5=~}DVm9r6KH>72_>ZEH|BpvRp+1(6dZZ%pUljg9 zGxh6-=>LlcFybM$U*bdW?2!N9@Q?Yz!YBr~|3e1w5id#b9~q#jQ2aUiKM3){XuI{_ zGQ6DgotgS|+qG=o|1Bm&f-s){Eg|Z6&xBFJZz|jWA7Y}RxBU;o{^FUs*b9S^Epz4n z5YuD%g}+As$3nt}hy==p@`D8bn@wN{|64)`(BFoDf&beOynY)390h(G!f!+P9UA@& zXTL+kZ$!9EzkVCSZ$tQP2!GW9zjK7&Il}K8;dhSkcW2@^BK$^#--z%V5q=}WUpW%I z8~M9x_+2&pt{VRGh~GKFZ$$W=BmB-0e&+~(A>!{G;m`ft-=X1mX!so(euswNq2YIE z_}5XN-=X1mX!!pvG?>v`reqrg6|*6H8}%3GEBm>%`(sL;JJTyU1v<6EB>#C(&Mw9a zBYRx@u`34t4K3)V$3gX`+allUuBSH+=ZMG2y5q z6KlNmKg8T98XZl*Q)-?*hs`t<9Bb|U95qU)Jz9wA7P>f12^Dn7$tycPY!Rw^HTpR_ z<<-B}xI9NEya|GGi86oCCgm{u0U3r9Vbi@Mvx(b}_;89M(b=nz@WFbl8CBdH79D$NBD{eLG9} zTGeWPku|uGCMW*SlYBka;FMmAuP`$BR$laneSz!b`1JXH0EJPkwK$3b)!`TP{K|#s z5~i|SE=x(nx}IkmW_jV9jQ>TTaAGl%PUG-1Pc=GWuP@(GXrimO+nBm}S93Xr0f+dF zO7aPZjdbFw?^FJp!Kct^5AV^ooXi*iHaw{D?)#$&r*m`{J3<%5>t*8>#hsjHxes2m zpS98@)ErE?QaR7P*F9TNTP>(R-#p*gv?v`^%_*pJpnjB2kDmJw_7boUJJ{0E+5 zsqJEyz|S{?%^?&qWdqDXuPu(Y=dGKzX;pjI_%Ff$p&`vcm(2|#&afuOjvtGyZheQ7E`2a2p@i z4Jg#PI3AAp7KEYgFztTOFDG=C7NL{fABaxr^aCzS2jCmql}()jHevdq0-Pk8WE;Cr z;zPJ+{J3*xsJx9^>4c724?Xg@VW&IYaxeONdU~u%iN`NayDw^tCEsuXBPtIJraIs9 z?Z2z~Of>fMX5C5pYzjWkjg91#^guz92*2_Rv3W?%*4M0z_^Q@>F<}MOO(dFb3v3Dg z#Mym=7bo=>CyCy;Psyye-jZ7!b~)DVoS$sUwG1$$>{+)^kRl&P57(r-`3G}Nar2%W zb$bfdQRX@8(UQ~ZFs|?~r+%C{G~TLQj3?K1i+~V$f9@7MIbD(v%FZl2YRJp%kw8Lb zvRW@4uA67E95qe}ZmV}$&9C0zn}UUmsQ4NT9Qc+^wv0I>rLvoT%uGryec#&DU!s&Q z%@8UzF*Tb|e<=0Zu8Dwn!d{|r{`pn-*-iUm>ORU8M|jJc z)QL62uw^zXDLQmH%*+oxCck1PV5J{Yz4rNd_^iGDw0$Pm370|T#t&^G6ldR7t;iE^Uornl)0Fce=UdOmGRr@reUNdvsq1Vg^4}hhE9v6$trZ@UUkDsb z6ffM>(+@gSQPWn*&pT>h8BUjeejjyaw0|~KsP1?Zp{?G(<(92}>%koJFV2=m%_spx_RBzDl#)@1y91K0j$A7ff!o7kD z6(sem|H|q~!Kvl^^$q5r?nhv?_TMOUiv*B91k~?^jLnEy5&NpRd6HRvW7?%fU=lhY z#Cw9n#R&I!rMUM){AauUP48)g6K8VI_QxSA#0(mHYUQ0hVrR?PjZPFwhpA?R%%`(L z=lZGc9FA$huq9D99Bme*4OiX@!`oHP z61k^2akH#MIa_jN@yAxn39 znSGjR)`Wm_@Id>kwj(#q_*IzeIY6RS7ruK|Add>Hr9|mR_4L_B6RB=&{vlue4&Tl~ zr)G^4jIEpBwqJ&;345k5Up;NPq+eEwDo}*^W_Vd6k}TEPl0-q>@k;eZmD$g)KCn0a zb1|cek5g9- zdoea#_o64Sf z6e_6}zGV>;`tzby$oUMU3xungNI$EYSzxvGZ+@o7SVU*L%=pu_VQLkf@l79f83~VHP3`>>;9SR#aV#wUrHe zlAB75t(rDA6Q|s?W8U;BO*l_=H8F`@3<89^M6o}hT>}9VfkPVQC zY$L)NrC&`LL};7V89}yFK|;b5N9h~h&Emf&6H7=qZlG8yBZ)aY;`UFWVH3Oi zr#JI8=;8M-zlUsqxHP}vw&^-8Es6DykJe;+(`8rxzZ)s)AtfbJU`Um6^u+BcCScP zta9crKKTa%`g)PV{_yfT;)<=W*h+WqTTE0qrDhqb>EF(@8v*dGX5I1Gpq;TSOsF}9 zb9ur^Mr(}NC~X}UZorfCq(zx1-t+uuo74fOS`~miqo3OmmL1&Q)X%vbsqk2SSjW{o zn+0qB`8{myCQ`rN!-276d^rx4TZEO`{QAn)>ii~QW0gZ1W@-3LxPgnYefjjt%wNM+ z{FA0%ew68Z|DHHa8;68(>}Xlev+B6f*tw?bf%g0lZ#>x+cAknXN>n6ddU+2qqudUz zi7I#4M*mJ1Ux8aomyKuU;iP4gd+v??eJ%PxN%o55QdrBaLAK)4?q|Eu2V>swWUXkQ?eYM>}7iA~&8Qk83#-6|E<6|%In z={;|fx$2AMujrFaTwmqKaeV0}6z|aVQ4S`lx$g)_M-2QWWGIGm+k+GrzRq^KOLxE7FGRO6?-2TK{hE~ghR=LXVk&Izk;dQvgyX~&yu1})V(JeaLu+w*4h2hTH1VIx#s*A%Itzey8&tE0XEGE z8&x+SsEA!P2@68}NA}}>#(0QG^~8Sn8Lp@79(`_Sm73@E`R+vX)2Y);M@ewOhg2t$ z%W?{2uVn8Pv7l|}1I&0hI87^0b&uNA`lb>tP8W-Y1|;d?i0~R?t-fLs(57jN-Bt|_ zIB1G<7&B>VG2U4eJgD3)tCeD{yhCwOcTfunm(b4iOYDn5X6wuQY(8IZILO0Y*Ooh_ z3Hv@~Rj~M!uDM@%8PhAkw*P?)38T*bA;6BV$OkEz#hu>cFC}7S6QxjS%IV8f8?N1x z%{rv8T)vb*rDF~=mvfRsG?BZ1efblD0Y+D@DrAsBE-ob0!cHLOj647&JT$YFTH6#SFo{tQES< zDG+ZZR25>+fg7KX*Pm%Bge>Og*PkB`6s{2{*DPD?baFvfy2T-OoC;$Rx!&ywwQLIUqGDwNJeNWkQXJMf!yZ9S|bz|GyLA>L}tRC z?_m$?)2oWBBiJHRp1zhAg1;mRb-IwNRHHijE}?o;p>apt_m<6fGP6@o-OwM4d6w?0 zz%XO718MjarvD13g2%?Z)Fg(XElD%u zgYdv_9wbteVGOd(+KsB!fM6JFoj0B!cN^U96?IC-Evg5#*s{3O8WLa@IoldaFxHf? zh?hxbg*Q8IkziY=_0Nu$Qp;#R8E^MNxk28(_$k!nbF9vF^#LUqxAr(%d8-b0obUlg z80mQ6-r&q8LIC+Dqo(>f#pZ{TZ;T+vKtAqk)Gi*59lE2RE( zySR&Y6)0RL17ti@7b!x=DHIMPx6S=UYSJNM@@iVvO~u`U?Xyr2jg4dT! zpct{uK{r2E6*|FYl=F3$zMhDlp_RX|g!5PSVxa=%Du29psb8;z)vR8p?s@m6#H7E^ z8l;*gj9lu6qun0+hqIxi=n}?6-9j8rdS~AP)OOPJX~-n!lkp+ff+SM~>*#)O! z)LuD{k58!<9lP*;hJ7^>+=<%es)#lL&;5KLrQ@#*7>laG1 zvHuAYb{TS7eb>oVoe05@lC5=1PuB{iy5qrubZw4=LE~>f($VNIX+IPo_0S+QZ-Y#nG-gzi4dbSl#1 z6{tfCo}h7;_JQ|Jmiy9{^}T(znGluAlm!ovsSbsYoj{4~v0t1ZL#1z!rA#Syd+H{) zQCDpJaZdfy#b!b@*oQB>MfB{FY6_aGg3X`Rv$i+gH(lS8rH9UoJocZr%%7EcyZoqR zrtpT(q0%vMcui1mT%5Nt6@+!?u~NCN6>-$#1?TA652`F5BIEJj4{!Gw!m&7FY(qH~ z|AAxaB)TG1^P$AeNOnBMYBA1{(s9BXSeWBj7DOneq1)xaxUq3Y-RY#?*;s4&e<%6a6L;`KEoY+GUyLqxdI} zDM4pU&z1R}=T-xHhOLCMA4l3kXAGBqbI|r@MFr7ZiPDpeYBlA-JKAFAc_%l1K1=l$ zJ9|W70ZhTQX?}*Mn5zyJ3{o7eA^{znL%fSEJmQv`i?dCCxi73B$`*{sfhdOE*g z(8QW_UW}Wi)CqywPEz#^o+S3D+zXy|nzTQ8zOjsS)%L}2Z+T812hT`nfc{=04ifBh z=|?$1o_U-iTqXVq!fC^1uJHGbuPT1Ng=6s?A?%J4|G8pKB&&@dh^1M^n#^;A9I0tt z7jtswb4QB_Lc2Ypw1fdh27p1!Fsj_Fr7qB|ahkL|=Q~*|4&_;hbfPu{-K~(IIRJK` zx)q8haR?}G^ZXdk=x<~ zD~qlng2zkL+4)J5RV!GUC3ipB1+b>F)1%pe_yA0DntZkbpa#5I_K4uyE{KZhXz+Ys zUhHeYIsjBL-XEaN#JZrv26fJg=1t?qj-Dk{pM+enHI z>V)O%SSVtvo>7aWeD@%$&B`m&cq2N$(HJdByx6Qk2Geb7 z-i0-wQ!&ei#xj`aDsjIjQ?UQ*T>vYYDQyu<5$y#ZpK>%x9ksQDwO{t$UIY^j>A0?U zd!FyilQFi*^og0zmoR@#3OEi7h4&zp%#6C@wRa*%Zs^9Of3v7NTG)O8Q_Tk*N#^xD zT|c_mnI`}F{uC`(s)kz@MUr;YgZa9F`p0XNcKtG&pt&4C)j32ZnX%VTRR)8I?u9r=h`ey z+P9arT@}X?IrK&S8}0cEc{_L$bBq=rGE)tdH}fb;oR!Pt^l-U9BO(-@<1^{Fkxoa4 z*K-Gw!oqd0hj&f7Zq}~eQE7T%E&WD#IV?cL&6X5BjLV{O{u9hk&RMFm=KOdyLTHGg z<-i5DCYu|xo#A<$VUKI}Az!xB*|CXM26p2yKb30z7{({T{i4t&?CUvHmH{mv?ny_! zlMSseEx0f^0KE`-k3O0>bs~oe>z2c+n%F3nTljE6404Q!JWoV%7&VB1%YV-^59v<4 zN?rvqyQk-p&?4)Do4vuq+T|AuNt9RrFg}q0wBc!=h5)eohD5-u?ex{lhsrp_`ETai`hfcJxNamlwwc7ssHXViL$w1~9?NMQz+xK6&eR&V3ZWvL z$2#49Z=!ZbvNs!)u$Cpg;t|>o&uW60bh=e=D3RcJKfCARxx?kjrp0{_2FETc1kG309v!Ym~5Z-l%1%Lvl0o)L_fv}E-9;!e{jp1D*ybT97bMHZPBnq zcNN^wWS;Li#hdjU=hmaZi`d+N-Ts90-6_}2U|%TBh+jQ=sF?SP#G~GMbuk9jWB(vi z(BLPy->4`$Pkm>qs|4)pWajZ+_(_;r)SvCc`{e#~@qTyiDFz%ubZ!n{=+`)GvRFDx z@!EjiT+thq7DS;b*pI5{hy23g9k3T+Z! z+;6%rA4tLyFw|Tuqp%3NvhtLlZfP2lXtJ|nhWEA-K!j^ z9wAW4XN<;#BhQ_fX=xP-kHeo?xjm~$XywBa@+X-1ODtgW6d5G%ACqr`H+B$$EvgoX+tP5(=8e7j4HxU){wk?U$uzz_+?_NveHodp{SRA#Dap;)!7&#heDQA!gKpi|eahS&38eHvG}Cqkm3XM+L|Vj-vi zodwV1{)F`G>4Un1w9g_1*f%D0s2;7;2`84LOxU}c)s%d?y5}MZUco_61gL(e^aEbE z4(5UC3)DYPjx+y18NRO077m{(Or7$D6>JZHTBdflPb$W|Y;B`4!5;QqbK zc&RdT{4JU~PU`A^B*t)RepPwT3sD}+dvi>=Nd3C??+asZPzT|BXMObVFMcg^gBg)v z;CXQQ)b(Gt{(UhVm2g4v6qRIS+NT7oxQ#;ZQKAAfMJEV0djWL=E&ig~z@2;8v3fj}>ozRuM zuD>~43j30#CSWxt5ID24IqDgB$aC_gHN`>_MOI-v+4WO8nV~FsLK|FxS7eb$4#Yiq zA8YKF26`@##$_nf-T=Hkd%nx*(G|(b%GnhwSl57%#p}G{$sus(*H~}?jVA^hFKay} z+v9@s4!lZ*_qj1m5D0dn0rQH=8H`Tvd3xE6`>X7Gh<~Zd$X{sS>X?`^Zr7+6AIbRFz!g%kqu}`s!CFU^9%JLHlDLi zb7EcTYLZ%AR3R3$c55P`y5iwrP2m<)Rox;zu@>?qL^!QI9?&N-@pa2H^HLOi1O5T@S?HL=NYkUy{}xE&}m7*+(|Z>1&n>tK-ek63Q%b}Gwy54Keu z#O*DJ_F^5*bI$b=-)HwdnAgg#+O~`QN8Sxy@4*CK__LbD3FNRFB&>myUDc$RX-cRG zXT?wWtRu?_d)n%m@|wV|7t8A7$M%%I8e3TcDb7uUq*JvI4?rD??zK0<%qS`_n!15?f6>KUK4;bA%uBs+ppv!5(x?a6 zDs}8^5L)KvXV1*L4x$rjzLiP}z{p;BW|I}o%=x!5!zB+lO>a#Uul!SUIQJ+NJ-lX%pTl^i^eVHysZ&f(^AXU>qPl_^xbON?5njOrKqKNneq zFR~+=30@fWlK6W(k4gz^nna3|OO~NZB~4~PTVQCC0y8TyRUSm;<9E1h{&u??vH)3P zgVw>l8`Mx6KASf#V=xVkK8>0c6=VgAbkco@u)eb_FIF!>!c2FP?ZCCdO zZA3PuF()Rk2E*?qF;!h`obkaSkSZ;7Jt!obr>!_`kx9861YS0%9P z2b3X!_LL3yS`P_gph&6Ds2H$S2(@jS<=7P2yyE(v1IN`FC8D6&fNL-ew+lSXYvuIH zb}!l(hIi#9*uxQ^=OpiV_6`n7YClThe5GHdy#g!l;DuK!vkimr)q=zmyH*dGu{g-& z7eVlpOngGSp;g1aqG{LNmN0)|C{0dEg(vX5{gB6Y>~NcPUXbve)ZVDq3hw_}Pccyz z%RnJhA(}TB0s0>Md#tzxXNA%CG&IMxz1XX&0k7?{hy5@ngR6!PsHSVQ`XRJ|_#F?< z#z=YkqD|INMRxe#6A+Pt!OP6Z!fkGW31~8jUCHupWMi01>GYcunPSy%7)k6Rclfo${W2*(a>@&sm8zNP-?=RZe1$NsjW*i7J)fDE2-3~eTVur zS!(Z)8eC6Lza-12=-FO}J|X}jYJ0M}xCJ=%a3d{$EH%m|lCz>I#{AIE33 z)DbHQMFWIaOpgp4Ea32(!-wBeH!tN46bT+TGyXayAp*`}Sfad?b!Q9X?IT2**D~!D zkXkKqZZ5F~0hAQIBUaF3zT1S!^B8_Q=9m9#13+FGOX|H3ZrQ~N#}S#%^zf|8#t+)F z3{fMYkP*J);aHfXM>4yv7#R*KEs>q#Psu-QbGz8cPGGi2RrWzx9I) zaPyD=W7ik(BGjw^L3^4A(_*CjgcY@06Q*UFxZr;^SGcO6anu!NtkORs+4?@NF%`fe zb%O<>X;s(K;lwW7vyB&smCAT{@#%jD*&7HT!$P?BnH30PYCQ<>5J-Z)p40H?S z#2%1@D{`U8iU-L@__*1*q9DdVHXRF!NczOl-|EJOxB`A_8k$@7(%;qt4FVVz6q!u} zPKP02suP)dp=qjBD^k11@K30lO)TbZbrg~7yy7H>jRtU& zKXsER0mu+|C?72e$Y3Qbq*so>@mtuf2jaYFQOCoy^Znvylcg@6Z){&PV!KcRo9so- z`x{4v6E(s7$%og+Kw1ioC(QgP@{A~ZQC!KpS{_)vk9V>p3Yw}gD@UPd!R#+%K#2yR z&5^RG1&rZZl^2Ku?r2M%s)$iasBqH1Vc|JAWJA83Qo9ZtFoNaH~Y=dF$xOV9mrI zSYN)tN{)Fg-K)doev5ViCjL9AtM5TX1M!_BayEY$ko}zjpjT}4$_fO~=oy?b1usH` zS;Li-wEt$RN)gy)__t&QaBGPJ!28~qpNVA-@Rhs|DiA~@@7vT7WI@(LQhyN#{2Zjo zz|Y~}qroC?q>%Y%`jA7O=z&B5P{97pq!_3G=@BajgmKFMHf7-L7};=d2TsuQ0zAv! z2|=rl1?gc%+5Kt^u$(G*mi6*ZyRs1x{YU%oaQ+)n*1T zFw@Y>{x^L@fWDc`RW>+%ba48tU2$YaPQDLmYjmR6V1AFak<9i5Qzxn4*<|sNLW1|1 ztBmHmqc1umIW4NQW&^YVz9#UPyLEnhbtQjJd<5{wTOJqwPD-daJfp}Ao2?R0^Z?%X zL|kfCs`ZffAy(b{wP@>pI06Of0L%^ZL>2E|m;ReLkb&c@d|%~)6UT60>n~q6pAR3D z0loQVHae)1Xd*6f4`E%5?`g++H|$^kGW$Saw%Y7BqQPk@XdMAC`{`>2({`!fV_9og zpi2sSOp$HfJgu?EhCTLgz@%p=bBfPysQfy6bw3jLUc)WNT95z>x5GKG(Id@dqPER$ z*S`!U>~{eQ2Xwdol0PynK>Em}AqBudi8hF!9M(&*_H^B*u7O&G38*h+&-km30X#r; zfF16Ar8oH%&V@Ar+tym5R~H%f`GR67vPkoax+Q>D_a4WpR}po2UQ z!dI1DPE2bKN-a&FqWW73PI=7dQG{dvPf5V_I7f@@J344jp<lbDw*De`($r*i(vv?Pcz;nxI;*6ErcIALzNC<8p;4po})z^JO;V((m z$jY&Re|hK$Z2zpjG4aAPsz0$C;S8SLK}mpxXn18mg^z$g4SEd_c+D^S@>={40!>j0 zB|_ix>@9sU;RF=Gn_j1cH9+nGIun&n%b?oC9j0BC!lBdWgMSTQNBARH<{o04EF8;d z@eAn(FbZAv<)f7TXrO=r7uZ36q_`q47j|K68Nr*L6o1#6-&sH%!;&osUoMmB3PMa4 zudj&%Fyaac;Jnlf^eyU~B90QzuX-7rX{O11@5~bomsT|CWlqHO%&09$(sp5J!a!dQ zu4hwj_P3WRDG?E79mGwOq&mSd-zK=!032|&^k+hOwvth6scMZfNQsnQa}zw>_3(~^6% zH{@gjZ!a86fUN0^?UTzd-@JUMqh`eGjeZfP^XZAM~& zYC2qIi2hHb{52GQu>wY~ERS54=IaFTLI`jR@GM@ziugaq-vn6n*TIq$Qee+Z>#>0E ze^Zfo0#6IL?rA&*GI>M{#wCW-c>JA6{UHg!e-emec<3q-@Hkct&rBza`fdK>xds5T zo{3(<2QM>zpoq*Z-V8%pW~@Ie@Jkh6Az~69^}DQl8NpK5@`W#pwCvG~G_FofhYNEK z$I<^(z8a9lFPBio8hA#u((6q307Q`gU=RpmQb=3_EChW-6v=M1VKwRT@*MVU{d3&^ zYcv>hwNAxQ#!@HBR%|r-4@PTg3*j$@cF8E!xI6^neRpdBhZMA*IZkdNs^pTX*hU0X z5UC%3Z69WKS$l8vss3=gU|7dCIFD0j(<_Q__*StCddytkzYH6YL;ZLTdOe)}r|= z1kN+v({s!BBkjEY)pCKOohL@_DAi;SHk_{|H7LkQ;rZ6uWIvZSf2+4VhskFPt^FA$ zuJf5za*zE+NP@=xoGnD$j>y0F4bIS%OMg+dTteXiCwZPG*1J#jr>hqRn}R=5nKy(& zupLYI+9+xdaIxrVlD+QI?_V#XR+mo^r!{>g$v%=?WTqGRfaIKUNd3Mk5tBnI1I6;* z(yd!9)?d!*!G8eZ6hB#8>mFb0>ah7OFro`=gYI^D@74(dnDF#}HiAWhhSkQzm0 zR6O>a2Hap^NTgQk-Cic(@=+wpR*ezFQc8Q-@^x$peM&PC!qGC#>%PBx9>kGxV5YEg z_gQHU@vF(zkGYI0*%x>& zGC});(N_E$JDZj5YOnn37vicX+V`71S?&`2VQS#XeJWnhA9RAV{(j|OPNx0WO1*MsOg2T>=Fv8g^R22X`*zDNFF5@@61p+HxP`t0Sc<9CV~!bqmuU4KTjD)R2uD=AA#wssI}I;;iZ z$5@x9D)o=qo1UNUXKzoljFot-#uKls3}z1%xXX5~xYkkUb`m$swa1jb+*y$sH+!jS z{z<%v&D{*h$J~iiaFfdeJo=BC=NFUvwFIIaM^}x|P zZZ;L2vS+UDr$CO;#$e4`ds8jfS1mu$%PZFRoTo1;V2QXqQ`ogU)s7L;x@A0;whFyn zlPj*{Z%4IbhWAO>cr?p4V@vhgbHg+V!l3o%m$e% z)vQ)=KbcKeAurF_oYWW^EjO6d$y=>*$m^M4?nG7+z*gNjDmXLh9Wz%Vv9z%vvJ9Mj zbt2MtsVb9u??mKKKgg{$JYVVSoKoP+sdw#{5U8T3Ppg0EUsS38Y`W8a)z5M%+9ppT zrQ5JzTD>sEBCi9PZ*FCXyqs80^Mh_8df1B9O~FjG@9MH)Y3|v&GwW@)1-f zMc?v4nS=x_U~~E|_-mqv)@)nCLjD*D+Dcw5uMn*S$Dd^ zS_@GdF_K6anm>i+$#lidF4nTY@aIw1>);WP<4KGHswc*&S{~#{kk%lENv4?&x{KeM zwAfXk&>|w})J>>aney!BU_y;@RmsuW>;C2%+mn&idh^Uv$nAL2)b<7~?5q7*b$KRX z3PQ%EZ2DBFRpPPc61Kb>z7diL@_0baP;_g~e|+{prY&-`og6-S>(N<6!Fs${LPfex zOn$PsROeVDXxZE6R-NUjQ#k$LL~O)lyZwmV<7De)^<0$b*p!tU-A5(dJb^;@{7MSU zzqFG05Myo6exa@B#jzheHoZeb8Y5!VgOqH?Go7dEiM3})Fl(htcY663pJv zJ<0AI4}f^?pVZggZzuOuEMht06BiP+pPAT;Hnp9J(-jXA${JZsU=_HaQsT&H$$w_z z4mn)#?5;dW8=X4**p;uT=F}@Xy=+Tw)XTr9;p3pia_X?*qE^aV^W!_`RL2`zPFFq4 zm@;BlvlENnX2Q+$m05ZiUHEjZYeLZ`EAM)PCF!OBdp>2WmO`3lbJtdLg0dtmWRgo@ z=bQJ%**tAonwiHcygAFGj-=7ts;9_#q5DtGDN+A08b#`JuD~0X#FFle_|@I$-mEgT z9lLSW&-+adJlS3tzIiq_1;yQ()C%_$aPPF~CaY{4eqD7^nzvYW62`Mi%Nl=r6vUDL z;iN2obuLWPOdID1{$;cQzn-{))Wbxm(GGs+i5y~HUIy|0%yLY+c#{&dLzLe%YKW8M zp6Lb!Dh)~Cr!3+3=?jnf%(fFshYpksrxw;X*J@AW7;bv%Z6_yi@qUKz)Tn*TRvS(L-KR>HHTT5_D*Ao(#FMtP`}zL3*XrHPvg~<3yM_z zIYl`)qjvSVMK9@o*`m6bNK&Z@4e%xe*gdR5*12zSy9`_A5*&JK; z-h2G6WAqyD_vdr_{rkP$KL5D!bk6g6Js;O~J@);6)mFroW+|f>cNHvW!Uu~9O2dz% zSBEos+h=pv4L2>T_tFX()-f~}s7xNt8>rgOo2ivJ1%DQFw?x?AK6~p%w>=-n?5@El(;0jFgSF8_HnW*dazZl+-+&zRlTq4 zeU$pIeu1sy9n;{&5u-fP_Q{;KC0pX2fsLM!b+epd%Cij!ERc{sHt-aQL?Z=|p5h!` z!dHbd`)uK*`QW(GX8Q_xSt)z3giV|l{ildBKgV0lJc-}ah0RTr+arKs^a^&lZ9Tx> zx?6TK@p6r;^VEWs<)Fe`OuP5%9Oa!B~7n@oH1_K~CF+9EL#v5(%#eovFKhBV94 z2D@>Gs4In$O{H&Pu!Y0gxcNeFlM?A#Ee)(qqqHxSxs|WW09U<5jLT-(zEQDDbI>|a zp!&OYRg{w{^{JbdP=Nk!FGENVr~B@gb--pw7hQz9Bv02Q_zj-m@*aZiH>kpsc9O) zY&K&WYpHmC+1pb?>Z&AKcQ^tMR(L4P#MF1ZZkA;1GshBdm0IMd-JufD(MI9YLUjRhL}crK;w z#lW&zDx@qhzeGS zH_c5G`|&YO==8_oVOs}Q)=WNU$5B|X%F`aUtnIl)EQ*|Ag~{$@S>;8HFqO*rh@)+n zOx@$|bd@db+gll<6e@ffuT|fXk=NKO ziBmO*o6lM3xQx#gF;Cb&BR3R*44~RomgUK zt&W|^Z#kP*xL1U;Fh*IS4YxPk<1AwXfZM>&P=WY9v!c3@#=KTaY%6y#gs^whde|4^ zbmFc3dCAqiK|N%J62H|%)1h5yW!*9-i|YI-$lb~~b}V70o?fqrz41mjs!ZEHPgMz=(GHA5U0Wvx!N%rb3_F=EW#=%T#WH$XsAvNGBm7GC$=E3Q%MWY4AS z__I^<)WY!f=}*wRwy{k)8n(mbRkX#oD-1tH5BG{1*6xyYcc<^FK10suGO(H9&Vb zerp-K``3wd3|J;kgR)rXtAyf114^ok!siD5G}F^{lR+z2`pQIiE#(qd(7shKqMdrC zY%@r(NO=@mmfP|gt1$D&mi>opVrRl-|6RlS_wi{@BMVy>Djuj!D+N5O%D&F{Vy$NC zSXcATSVp~Rl8k|$Yj{#@h2Cwe*uef5szm2TiB_4y3{0~Xcroax23L!d)nv70F!yGk zcS)`a;%d)u>}rIJRL_0*zR$T8r`(nuzpX)E;$T~G^N9DEn*T~weDB18 z(RMk+IgfKFW4JwQHRC;oan7*KUk1eq6Z?(C%37)e(z-ZdP-6RZuDg1w4RO02!=#r# zv`4U&(-3Z|3w^j*8 zQKWML6zLm+>*I=^Ja@X^3bSWXZ<&~gcKakcFBQ(>$Ca40#|06rTjaubuP#;Lz|_|} zWEWPXzi@ilAdFSUv(kHcTE}Y6)XJ&R^Z7CF~QQZN27oqD%Qk z>zlxp*V|OYLnU+L!^EFuuiPI=6>p1cEm{(6?xQJ)8LrNmH!&t#qP0+k+e@z5YBSZm zv@+#obDZl99Fn|_MfNMHo5;Z4)L{N?+9cdB>MY`4#dx=oA+&U3{@d3AxXV(S&9obtr1X~tvn(@>tF8O`OA2A6nwW=HsQU(^5KA#j}XzPV2%I9$0r11Qs z9ZP_A+`OxOx;Rp5gkn6?>!2SB_SlJL8?4zN&Q^wfxt!D8OwjywS?9ilXdc7wL9rEZ z^3vf5kVV?)t;dj7Zcaz!n^z5N>>wTrbMPIEC$Sxrth%KllyYMmXL1pUMDgNMc)R*`3I)sBt^as;YJ%&9};k zG$$!H6qb(4x9Esc`&g*3AeAko>?-E8A~?!CQ0Z`0>~v^hsuV>aRQV)y_eQRhMS~8P z`}WQ)LW|LShKWb*FUmODH?9uvrWt=a6!$*a_Gt3o3eHXB zTlX9XB3B5_iJ>Q4j*e)#+s?Ow7kZQq=p8q+v8wB@U(r5ww_&a@jf7$Sg}qnXC{YA&>mB+Qq%l zaQtYgjVp-I=yEro*02L-D+}{_>G=Te&oe|MAxLn`DcV z!x0B}&0@Da%Wl?eLPRR{m( zA1Qcv?RMIz&EbP*$Yr1V~0iks}X_+6xJ`|9?TqCxJ0`;izls57!B^LUit z2*Ckmrk$ZD9Mb{5&i+c@tbc~-B9FuJ!?KBdI{1Q8RtUtkXTI~TY#Ur%X=Y!M_hc=D zd@(b|4uqw8qY(C9=LvW2t%57B(|@++x;2fdAp zJ%f+G)Y3f4V&5@Mx4~sLQbfbvlMku-2Dvg4lqGUy$0Zx&if`2%EKo}qaGB9|2fdKh z>?qOp)~tw4Io+dZ`G~D}R^gg3pc2N)g%w&gg8hZ7h zReGi;O?BZdcTZIQH2r2iy8I&6;2>ArpX;;Xb%|JFYa<5L>rHcWy^Qty8Za%SQ)YOD zvvOz3j}?+yyXS0KB-h(4y~p4UeQnuE@BH9eY^TKxv$UFSQz`D?$qq{-!)UwCf#}D# zJ_f7<@86KQ&CjS`?EO7O&zDDncV-zl)cgAGCYz)2hT~5TazoYe3drxkwR7f)WDG>A zq79XDSdD5LRz%$ctaEBr^K;;r#sB2e=jgvaakh#xFa6u1241UrMv*&2--=Q0aj?E){6u=DWm3Mb_}sYD6rx*_``ALjhu|OJmQ7h|lDEDw2GU9O)A{c&-W( z7@@id+|hK*S2?ZzQ_&WY~tq*K(vn%eqly*xqlN z9WIk&5xLoEEX|IYL~wUU2@dm4X>nvPkfk}WMOyccW;hlPXUuc3s!x_b(5jj5n4Qbo zhx@QsWSK9NA0bZGA(pldXQm`f_%yYI22x8`czXKuzFNx8RnD+{b!#rVs`f1n1XqXO zq_0%1(WEKWu1uHnwQ<>HaSPJG2}h}ZKvxB@da@&}*YXff&CxNyy(#1Ulbxf0pk;=J zhU(}i+9@ZT$?!U*ctZN?63&~5vaA$QcqS;uY&y!-k>re)>kS4$_I=Wzfh{rFKZV1| zcpWSs0=5i8@vRnm#H>bsmXGq(?d3z=Phted%hO%2MS{G(}aWF%CPaO`S8=j9x6H#2~<@650PG8g8?>ywt@ar`n zNSyKbTn0yQHg%HZkMQf+_1b4Ll+gZL`6#o?RhvG zWtJH;$j7Ev9eRW2b4I9lhJxJOz+w;THV?5%wF`<>JE zgsk>aZ_i&&2Pa)YK>_!3@h>d!n6KVKO9(h721X3V`uaLCm&IaofnOOrFR$}pv3Xln zUyep!g78~EWABv;*f+IZ&IFV*iPGl@5%Zz1DycYXSBuX&F2?&5Ip$Ld2hpGJ0}3+K z2(&P)j8<0}v_>I$3cRt%bi?}wkVtO84M+^eI#bE|@egpfQvpYW>0*LF&c(npQ^$hT9J!pH-2YU!={4Mtw)9i$eB}lJ^3<`>(UOOaMjy`CRR#)Q z7*!aTW7weL;gVeuFDn;q)Q!E1wJcQ%Y|4N~$# zXqcM|eR*z2`!JmCa!2Ye%F4n7fG&PH5d@16ciHMuHF)|4GzHusQm!0}ml1G)~vGfMcAoQO{pk*f;;7 zG#-4_l6(IKteuqiLe|!6JEQJ3uNps%R5_PejtY2+TK0btKHxTHzAXJJ15yF)%Z0$d zhfEhx$|(m(HjOhy!3qJ*2cVzm0a#zL;JeaM?7kg$SLjX{Hw6i0vWhClH|M&*mj-h* zO4?$1Rddvv`X2cC`6VPIfj!mj)_ByVdCplU^E7}M+hZ<{8xQc;gX zyxO@_;k#ITt3rw`GDbn6+(oK~wyeL+;xdGN>l$jEA1HIUAG-i*hdi)ldDu;RTdF^D zkv*r%H3?x)$gSx2M&C(FNqKd!z0{MXs@l?-`$$(^&|3YiyqBaaNazf?!`{g550x1; zN+~NR>ocDaC|m)O$Q3%i%^cA%gGH*}ITf>dLiwSe_vMR<7g!GB!D$5r)o)EHDlNa^ z#5XnZywYck@5go~$+PZGhTkRLdie#gZiR2nAx>nU@sY{CjF8Cbl}wbJaLkm-5z9~@ zqrI@P;yAzXuDBz2`zodpZ#-arOLjjw<}mGLKOyF{964MpY}cxG8PE9ObioBshR{wH zy&1AHS;_+kpk)%v%6!#4tr4RT6nP|TUb+3kqNJAVmU7PI=jdpSJS~33Lfj@6ccn2v z#95F>zW`Y@UCN!Bni@!3+ytFn0Dj6U|H{yC@TO`1m+`T0ZvpKp&e6JAxI^ku(XWiA z6Sc=%EwQ{i*@bQM-5DYL_g}ntF^>RL07;|c5Jm?q<&n89KxvglTK|CKRwMNBMhjc) zCqGsMhajN7m8o8+`R;v-E1cYA^&2(6(Ju;B@fj;XhUjM>sjKa><^_3pJUvG*sF%3< zsq4LIUX_ZnvD{s^T^*4zsqdk3J3aoO#?xSt7>kvHtEf|Ot$XNa*z?lc04*PZPCchT zmf35+3y2om9J!4*N1YZJ`vzW-6EZ~>%zH5qjyk67Rg4$9Qysv%gD;^Z609D-m1{ep z-gU>NUErmV)$h;>JpS>7gJv5m7qfY@-*zfYH7YGsVHB#o2Qdx;K; zR(m!^F7#wOzyKs0^ElsPxI7n=<6aB?@?c3GH?LX$qYz7)7QTD%nPG=fjwz02`Ux5= zViZNuyvSHy2L!x+lrLUS$HK0G$Z)oz{L0lEk8*6Nx|v&N3rLJRJ(VW9RRpe>s9VtB zF4sT3#s4ZB-@o&R8Wavy&wB?pab#^2#4ujk1VQ3rlO#8FXl<)-@O%nnB2+Gefcdac zPT+(UNdEa4*@v_OA7mfsPQgrry{VOjwZIuK*2wI z`MV(y@Fm0_IVW%}VKA2&2-68oRZ;2∾55dxN=S(;KbZbN#C^= zZBgEcZ}cm{vpU&ChT9kTNYN-5!^YB_ypOi0qb}l_|0c~K(32GuB&rv_!0l)9qtT&g$rl9lS( zKApuYmr$md<(k|`a<4HX^YxzVw|_O+sGGG8seTKyZR4JaG>q#gH!+y@Hb{y<8F|M^ z#2Sg9MMBn30A<=aOAcVGt`D*_p5xpg`T@KUxK;dpshXoD%hstV^hDB)8Cih32P|*1 zG17}nuxSN6OF+K>c)pDM=8|oXwgIc?$?cdfg} ztWW2Ka|h7MfWoj=Y5j1cJ-UeDou!39A_nK=0rJ5 zg*&YXmLFFV@4Ed0twrjkXj=C-z{d04x$H!KwRf%w-6}$p!}(U%w0zSb$9XAIC;j2QpB>F@t3VkdF?#|BQK-XriiVZ8lCl}xs#@crlg%^zU zwTN9UkG(_i!B&F4I-Ns9N?9TX&{bkQ<>5xKWoXy9`OQPD>j`9ns@F2*QXyL83K1)f zCWaI@6_Z5+7g4M^+h^2fgGNY(l zFb+gu{!HaRjX&L&#Uy;_Qwk3g-22%E7q||6x`07@r-FHMXm4bC;q9XK}`Zga*GqQU79mWEZ`L=PuU1ww&y7a1H zln=NX&Se&MAWT~bA?uQ%NzB`%Pe^Kf)t1Pq)MeuWB{~TUJ@G(*RGgbrzWE@hG&y@b z6WtTzY2hdm??mAPwWwNsKM|=qi@5lFt6{*^n7aWw*F^D(@u93K0G_4}j9!GHs_^@g@-DkM)9Nl>^D>b;h~GZk@Vpd0jX5bU zYBz7vE7(!fXb&(*5-K z>lHGpJgYi3JuCcD^9pUx*6PPhwFDGlC(9H)mZh(foE7bL-6M~wsCP+t4G!o&lG)4AJB$a=BdnJ2unvi z`i5WlR)e}rrml;xqB1s!pAagZ3U3c#_(6)?0n|CEdquDqZaXU6tdzqu(}7a;rDV4^ z7ok4im8-~xx%4Zx$|N50FrWQfTHR17C|clRU#CPaJE4p~<}Tis8E_GXu^FbhH6_AD z-qM&OzU^lu)Pj5Z#JYv3MGHX&j21N7Y>7)KQ>%D5yN?Ezf+J)=JmLePxbUKCsXq`E zU+uUH0?x05zi4WwR{=>stFa*Z(%gg@Z?^UW4s_AMhZ&qRHEx;q5Rx%SvBhxa)#!bA zMwqxW9)ebqS7Z)%CUk;pTT}wNS5W-xM9qlXnU@kawFNkz9V^7C%`Hn)wO8zQ>M~^V z>0%H?w?nIH`mT|?yxjg#@Kg@UCv}f@No-0pSA@8Ac#F#I~JbRP`9J(rsN1anzsZ;IAv;(` zqYPgwQhq4D7+V?Z3_8gd^V%YaM7k~8cO1JAOO0_YON=jPP#@Pp;ro8ky`x=+G&G9- zB!jk7A!~R#qx?kQU-fDC+5z3#gm7y*ug?ep@a&sRTE}IOf=$IVvb>8YA#a>Rud>h4 zAqj%>oW31EPtTD4Qa~|Ci|-p-W2wI>vD`v$jvkkc-1I?NuBp9_E^@cx*K}M%g{|dP zuKS7qcy9Eis6!PBg$A(Z%Nmz=rih2cuwn>vqoa`t9&IYpg4*Qm?y>`TV&@8u&hO_XcceDO#yyV_pk=B8r7rK6zUelt%hq<~CKgqDiy&(vP^%eJ%iWlZ9z$;)c@nON~3iTC~FSI8=vTs^=I=;)CLJ2Y#A0j;7 z$O0>8csxaCO6AFi9ufWdJ8!~rSU^Q-T(-hyMt1G!H#l}K;WAsEw-5ROrWy_=rLhlW zr}LdN-kSnhv;#byKC?~&0Vx=j%zd+9h1_ZD(>`7hHzyKMpb{Vw6Lqdrd&Mty z=qtI^9NJrg4@`xXxef%D)ujGd+#$b47g$MYX&b-cM}^BVC2F-u;cfaCm58TK1Hvzb zXv%Q)hoYCg5-h5=RMNg8lO6`G#L|JCs-|1YDm1pMW8XNV1WbQ5=otue=G4!;OITFi zl^DKGFX|klh#oAG!7+%YIY?QGwe*RlGdHV6N93ka0IcD$e$-NE{;hWInHXS=2L{k2 z;%Tb@Nn~4JE{FU&V0LA*ze@7PCbJcwZn|YDn&d#X%=0Ch%hmXp22^1Fm6QcmJjBcW zGdYq&%4PL4wy-R|LVPK2<}Wa9KS*+={|fY+IxktVPCU4iwRlx4fLWWi4>)`?on0^BbMoUg>Sw&Wm_#@ z<95_?pB{tFh_&m5VXS^$ZQ5$e@S{|Y#zrh*I)Qu2w!R3xmD^M0oN-SD^>Ad5U;$c> zsB@m_hBPKAE686l*`YnQE7D58%9xq_Xp$!0WuhAf-;KfU2DabCEvg9H2;?k(*FYjA zsIh(43qw^R5u!Q(QIRNgX!`mpu8R#SiX*O^^3(=aL}f+4tV+g#(r5LRF&y`+AzXDh zmNTt4=4om*3LDV}A9?M`Hzj;L%SvAI#Kr*DU_9X$B`Wsriv$sW!eX!OYZ7LR)y>Q` zL)lSDmaHIy($kc$YSRZ{|3)=&p9FeEH!Z~B_KM8U+=dEbSbeWl&{1MpJkl2A1|0$x z(T+A{SVp7dW1I(!-)D!On6A;Grf%J7C0hzih>;SBuqdCSra4+F8Ks|uRlR9psPi?B zY{q-OOXSZwY%$do2HjKW!z{pzwmaQ+(%+jxeR?s|J^^a__IC+@@ViF8`7@9Ge1Fzz zASri&REcDQRV0c$hx&FwGmRB5lQ6P-M2+9VE<-;3(py<9B??)T%uJV&%V5lX&haSs zGIo`w=S9l+KcVeD4X?ZxRQOT&=viGUr2jqLVXy+rBW`4h0$FSzpp}E8th6Gd0ndJv zeQ09U?g%(=LN7cR+N&k8!0f))3>;vRgYR#QwrEFRWHRV_URs&R)Mn@~W_uOFZ9lIR zvQ1rt8G1xeFB8iX$?B$l=ck&{@Ci&hI%@l&MsZ6@NbE0lmuW%_E|d5)UpyKfZW9Gi zZD@T#WBv8=wFt{N6J{gnm`-xVh7)=hw}x8w2XXH>3s~!mv_8O&BERqAb8W)M8ZJ1=vG1o5NM=OgvFbeg7bol~Mj5%r{TTC9Od7sInW*^Tc4 zAVBCN9D=?2geyal*?JT2`ZFr=2os#=lAe@k@d07L8lJ3p@P%=IA9sIPyNzX`Ww1th zSo2|`Ofp0NF!)JE3GIoFL^Q}~n$ulmG^ePAu4fLVm*Iev*Zr5?*YL0UlrrS$UIDSP zMG*onYe+JA5XH-5R`#5(d=eVm_~S(NSB`KVU0Sk@E=W> zV=cXVV=l|T724R*sWTlyd?p~ZcmY?b4dfY8HqqC6O0D&z%~(s00hzW@=LpzwirW^l zAdyfU$&u>&45|!^I~R^5a~1o<44PByY+3_;vGXfd>HGN-O*%pZ*E&lqMgjFEI0Qh6 ztyAjl18Up_FU7E~Hh>n(RpMr<2oBJV=an06wghqfy0glZYmxuHQLgDdDVzpAw&@t`rQLD9`AYO zT6sD4bLJSHtxtvZEk33*j*P7{RhD}*X1j~p{wsW)ds%jSTpqk2ynK$TQf1|}*C#Xw z+_9!Ko#S|*&#z5pNPS;&e|T8)Hj8Mx%V%Mr=OC&QoO{yhys$d{y$Vf}8xhbI+V`n@ zdq7C*xi#lVoHx6Ela`?AvDmjcrGU>MeBzTBV7_}{5ao_hoVPj7ByMo|0Q3F6ND(Me z03T=rMDZyvKUFD3?U5;?tKMjQD4M3YzrF~13*Z_G!ouH>c|wzJ+Jh%FOXZr6%QNU; zVWZ@)$1ZaS<0w9*9&H!4*pfsOuF#XP;E)-1`0srm^ngb=E;;Q@L)U)E1 zg%l0A$rQZ-IsN2Z)w3)Cgudq7h)48u&%h5#OJd2uiU08XM*2)2uUgaygaG2q0<7VH* z3XU0H6(kfG{nm5ifU%VCsa!c6tmK%$V4OD!Y{=anaO~|XMt5)HqEb#T2U4zR6Sk(P z*gld%mr_myQpdC_pi8v+ z!E&4agI>wAxqyIxnLbwiMFJ5V!l^R5Zm6X{R>!|E%_xiLmzl=ea#Y>gK0 zr)|uKN;`T!3em>!5L|{^&ZNL;;}?Q-9zQG*zoj$#&>Xl1;T~^(5o`r}oN2ug;ugVS zzAz#T8k__4ff?RXtOE1uGcPil5i&ftUa6 z2(@=>wFy}U0QnT}yH4p)XA^*dD!OX7gizL~C$(*1>A(oM%fAnf2P|rz4`(^p$=3tP z7Qj*c1+%pCwzf+?thtmm_NjEb4K7$@`EDTtfUTY!@Tr1Bqu97{@b&|=Li|8oF=0R? z7v+3$G1jFiqk=Qno~UlzzdLUl4Jw!cuQRhF_g^AShz*3m`~huRgu_P4@zKG|^mIAB z_!x-K`ng=e!A`pGj6uSk-?%dcnC*e^*InqUpSmyi7#MJta_Vg`RT}S5jcYv)o7JPv z*)mj3CqMl5_Szu#J@G!Xqvm20WOW!g3^sVbn5_R2Aut?y>?e93yB$_vojl%kzlYZO z`!EOmg!1%*e3ldZ_rz-s~6q6lyik22LKDuw7ylEpgm1P__p| z;yh_ZYKI_M3kfs5+5*@Yhe3h;z^0zEs1tFtGn!}B4Gb#LZ<`(vla}jze?Jh;kaOz? zT>E2LczZu_xp@MIhF)!!J4UYc+r`s)qnq0)X%zYWEKIOK=`?l(09YZqm{FX=CULqnQ{Lb6# zQ$6&B%Sn(HnIFhX%J}}sTCEpmu;B6QsuXF{uwT0G)oZLlFbBn4vZDx#zyOuWD1QUD zRd8^fnJsU~_2S#=<6WFc{_57i=ch~QF^WJus!R11fTReE3{Ps$x0)AUB-89`IO$KW z1$_wKj@~`haxlyeT)+a~qgZ7t4oHS8>s_ipOqjU<*F1)EmirL`3nn0TukAA>A5e>q z+I*82tJD!gsiU>!Ed@HVwmgXfs>wYKHR_*aYEX-fy9_{rtfVBRx3|uZ$S9w~@^!;p zxSOK#O>j1mcEKxOXj4Z3p|iPN4tWxtGz*g|mEVoI9Ua%9y!L(Nr@%6|tTeo58m|DF z^r^WVzsXxNIu?#W^rMBU*K4}Q*_vevAHRU`V{^Wzc{vEuA0|`k1?2~#&y|2g07-)6 zF}|eSC;`!YZ;nP$Ba^;D!OBF6eVIT>r@K9K&chkMf!ECph9Haqy&_$>E+)}rOFn9) z*{zbVA8ul>u&r(u$ZL1s&AFEmSYV$nYqII>2ka*3VO{IV#m0q8(!`CkG4YR$TU^Mw(`YB zal0Dgot&7raO#ae-Tk`R4>NqkPdZ-%i-q1FN2<%d8)ip-i7Q#g;_r=WbRRE?zs8?1 z^PRJQ;Vpi9omPMIY;r-$a1ZqO)4SU)p9Qua+MYhv;RPzh&#CmQjK`+lF4Y5j_al&G zhFyWhz-tr5@)5xq+5mW3<2YmT^n7~4l4orRz+bCUSrl)++evpic;uAcz3j?+0FGrV zq2sohd>%smS1T2^hKE>2zd(%0>Gn&o0l`pa%x{@IWB~wt^DE;c=%O0q^xUM$w|H<; zNi*b)-o;%ndD5crNr-sfBRQ?Qc}g$>XHVglVC-2F(51@{Rsk7(SBm!yqNG@wmV0Ni zM1C6t6TpMs9zUm72t!uurbr!}E^tcD=q|^5GoU+$^Th+vfJc9NIl#;2qrDFGTl)Uu zY!LK0!1b2k-LykyZjd8(!+8x#w=Ke8J;-uT;oCmH=y(G{D83Y|knj5VtmRE8mEdP~ z5r5k6Gao)B_bU)E@Wf%;CB|LUz-r|Op(=>$mSC-BCOCC~#s?&Qd1+}WNFo5w%s@p& z#Un6<{z4QKG2n%~Y&DGt&SC;%&_n<@4`^ff(-qa?yW%`=5UrQ-LOM_&RT$T(x87)cbTq>Y^S877BeZnD{2P-c9`T^?k9L zJX5aYGe$;+I%WLzABr$E(3BV^7F_(N1^mAWJP{vEZIe!-{D1rB#Wf*%B<5(UBYElK z%D-my^iLsL(7Whmqh@pw3jf#n{m=xaR>5)8i1&Y;S_xz`C@CTFMe^alJ~*WS2t?^F zKO*SAjpyRpI|Z;4XuZ8e|MYeKx0_`k8z%oxTqxK7Ph95<^Z!=Y`98t^zs-22%KlGx zU0Ct|$5s7@$fm%w|1@TDxhR9-GJzXzIFh|)!!&KpbVvZ<`P|Q=C`(94C{*1lLgIw} z>B2RMz1%A85Ro*XhUS%f-)IN@8u_MWr^vBW2ajS3i4d52A8_^G##l}V1r)==SF3yl zsib~*U05a|df<9X)XOLTy;1peAW8uqDxFz?;0tA^7e5^}cLMj%U_hcD06og}#G23j z^*_#K`X$dg>L)B{nJDl5H2#DN1oXf~f3jD`eesSLZe#}ahp*3QVM3_@sjhW#dx}fI zBwwabK>PQn!z2>2@e(6YlryyM+FkUkV_N|~Z%yK#Vgge0EI?_e-9<&KK9J%z?(ceK|A-^ z(tdO{J0ym=-*GaAoDTif+#xWFS4_u{+w`o&PNX{8goPsbAY zVdKwGc#xjaOh>Ba#g&T%L#a%$jT{OUL44E2ba;LMpT^j`)bSrTuVuj2LrSHV*R1H2 z>jn8#+S_2@Q?)J9|5|O>&lNq5kP;7rU6Cod_^0P<;C|IJh;aYMz*jMz#ayq)r)9>! zXchsqB!4Hc`}^3jzj!zxM}k5PLxMg|-<|>u+&XnEq2txXl?$QNQjSG|Tc=B6^wifnZ z;|scUk5&u)ufNZK!}joC!||h&yDazD^87$}x?ww$9@zaA_FasdUhU;UXGRp3^0&aT z?Z0j}jtq2ytP=?Sy7ZsF2@#`z!PX|eNe%h)#usz&XYAlPwfBHSN%~ue;lKU!;+iKr zSZ!gVZ^czFuKfKL7uP(ku7h>+k6Uc8-1R{4)$IDKF|MgR(!2hmg!7BPs zq5SP;&mVxLXUra9hxXsbb8+puH}Jo$0 zKl%59=u~;xChHDe85st38i2k?GZ&mw6S-dJW)CV|h9EoAYnlfjSjc0yMc6+$|55UE zbf?J&DiWFGf9){(kPD#I-I=K=4scLA>sDMfwJODS7Q-3ZN(K|8*^p?f$>R5ji*I|{ z;k+bNDi5GM`sb{7s=u6Ri-ox(K~>WCX)mK~kGP*49UdNLXJ>;JKhRad3T*|s`+blG zD%7p_11b7AerK#Zk0_vfkKzN5<~t8*x?XP~QZBOPK$F%Hg8ew?UHI=!sD$C^yj(T| za<8`KJT{X*^Ywl-EN_D}xD3x6WtvouI=qE1v0v5tk2MlXSO;4ulTS}t_ToLxmuM13 zHfT}@b#~=Ur4B7c0QZt_P95`oTw&0@lHb+g)H0M{#5(zx*?tVR&ugkbHsow%N>EP# zA2A)~rh#=9n-7Avfz-g~#14$+4L(7CJ~~G)#{uCLt@+zFJC&f1V#1d%zgp%7K+ZZw zHT79O`?9*kZrAgRU7WxQtVU)_vh3);hW?KmfgAv^L>LTVe`fyQLPHKD?1e{F0{zBp zm&3)~PcjZBAy zhMEBc9%xy@-H5ZF@^JtyAoMZxc?_xt+~;Iap1;s%U%V6i_nRwae+6S-F8v(@${T&z zFS-t=hd@;n^xGiOdnOuz#2ix9A-_#l3-p^UD%Tpcs+=5?r7@mCKnFL-%iBWM=d<$Y zQ9{pAGi86FW|*bMM5k>;;=pmTeDELDv7NCz_T`|Vhrr+w6$HIL3ml61kf6{xJS>bV zU&EqI zauE1VIyZ>fXF-!u{(|FT{tu}~8Qm;L!z*qxmF28#Y)C{*#h*H;hmbDH0VR~^j6wF# zdIfdpP!E`O3=)_!GxslGNIwV-~Yw$ZRCi94%4c8^w2W!@kCkQ7v|qL#bTWXZ2gN!YE7 z9Q`17FRHLiJE|!MAcRq)d+DeRKP_G@78!pfGK@(idIFMxsLcAph8C2g4k z7(fj;bea7rt-2?u_irMPBL}FoWCE*oIN^}UZAgm8JG##yA$pxGxKTJ$G19xDglA%~ijY7eTvHXSr9oS0l4&UyGM8)uCD$;*s;VS?fw9 zk9{V?D5;5YwO$ansQ=GMQsnYyRjz+d$tlW09{@E0M_2m^y6#PdcVNt-K|u$5bUX}{ zu?AY+d}=SX{+$F#m&Cxtj4j02K_c7K#n&&?gXb=q#9olvA%Q&Z-|a9^7oMo7cL9jY z{P-19wp6Q8X6wyMh7IQX zuF8W`=F?OTZJUBgxfRF<7Lpy&=$>J-e-GfNnQutENmf=mf=yy8M1p3-K?yu+DJvQNlfw=5ruOcaGyuFoWt(makyMOQq`XkHo~-Cf9W5EG z5Fwp^s866n-{8t!7tX)mYHLHU^lECOU9g0!O#agJgguAxXgvTQjQI|5FcU)mSOjt1ugLKFN%&rWAvCj-;Xb)J*^`>N_}S>5K+X;0W!#^4A`wYci&r1(iw56y zU8&lMs;40kSKgtKjWaWO4gSXe-JO{9$xkfGMTE(Db5;i{_T7^&00PP)sE;unYu72B z%la4Hn(uYWFR#hN|E7OVcPX}yN2K%Q;gc+(k0l6DH0|f$e}0cvl?9DNlY4vIo7~OF zO-JQjqrg=F#s2r?cKO8bm zkLTXESd}2?kMzLmE$q$ za0wk%0rX}Sm1?f0u}Xjvk$LWPsJvY!sEdn=lh`@t^@ys#Jnj(quYbOG(FuY5TnL!l z7Dd9p3SOt1(sP~gS2ww>WDawht3^wqxDihqKq!qNI2&W^aS>GwVS3~U#h~6iZOiG_ ze*Fn48t4N^D%#apjagNPLS@;-XmAQ_G|U4=%VF04ixh#*YnReT{V2CnmX<9$1Lc=FTCx?&S>159a$?g{p%eB2}-|7VzAEf zQ6^8wc9s(KvdRGM2a4cj7*-d}wE(6F0vZ;s+<&ngTo)4Jy%(u|+60wbN0%0pu22cPx0~OCvbgwqTixb&O4mSNYn*TtVP=ite{iDz=1rrCrALNnO1QS91~-Z zBN&2*cy@!qGP+gwZLXrh-ZL8ao=jz^r_F2ou#-E+@iY~|(0y=%H>Y>hVan82%y(oL zr2c8g3OcEyx{7!ewDLSe!%mAwV@eL(|uf>_G*U%>LIaw~E+Q0VP04NOBA?Zc6z) z8#bMo#$#VSVUqv|3(NT#^BrjX{so>W*1uF!9SHuYrW=(1<@>myahmmiv0qS8^jE=U zn?uC~Xbtte6>zA$O3DwHlsF;&RIN|_f$wjTAA^dp1E|Zxj>~LkOcr2B%x#ELZuP#2 z>%OH7UE<46wVc#5VX9gzb9d0~HP><$7Z>m7PjIL93<95ypdw|$B_DIvOjjjU#q_q^ zCD5F@c=%Eq0uZT?e7#puI-rM~7_P4#e3#!&#a}HB`0vQJafx~OCIIpcXBY1P6&OXV z+wzQE|M?#su!adj?L(E&KV+Y0s`$EsP$B!`e2axR)F8&V)uAJ837qdcey~y#52*6v zN%#TQ81!trgFs+W21G?m0KKpL8z>|#=giMlJ~0phXgTnya`*MZW#?ZAs;qH6L{RHo zt}ajFpf=N+M7T5y@BtHeBlL|eXzM zvtpLMp+b~5gy9(m?%z2A04U8OqmOjjfK~yzgfLcD4xyH2W!pKNKIoKWjx0o{aT$5e z;*phGE)l`L^?oM0 ziwRXr5E(CmX+Y8(&)V$ zi_&9l7WqBxOlseu$f&Cb9oQVYt5u7_!zk_&Xc$bQR zPfaHly%XU=Hvr zMzI;-4G5~j7zF^)rS&tuFXS3HK}p=T2tBco`&S)Vs0rOD&aY}Wjx2yqQF}40?sz@C z#z=%$^)N#U4kCzW4CG^h>|6_dpl=e;U!Vp8{{jt}=Xq|2XrV^JRgXh^hOB<|V*RbT7{7l2)k z6hVXHV{w(WAq3O!lOFv(LKd>0cSN$e)u%Nqs|c35ZQJ<`Wy)YeblRBQCjTFMZy6S4 z7xjP{I~Fj2fTT!_G($@_Dj-TqNJuEsA=1tK_f04~ z?|aVqc&_VQ=lP^QbKm#gd-ZRvz1GV(R#egYa^h9)mA5j!KxMSYGX5a5XiIr z2kugk{Q*1|z9`5f#zfw}pejxLv{iiVmg6M2w@Mq9lpz-rZFWI*1UUT~rk|l#)e(dRE#9F0!ezyCDoHKK1)*cG4Au9$us zsxbcSB>3dv1Lyoh((66bOOhJsVch>gsVAl^o zB{VV;=T0EFKOGAq)r5y%Sdp)%oQdX))vBeFZ)(jie4;K1up(zsHSb_$*3R#RE<1Dm zNY?dvBWN32Xl4X1K0WAg4~0Uup~b^`t`jf`?5keJZT}T;90m@Q!E*^R3^1s%lpqT8 z1(bROF5o>qeZIfcAxXlb2P?HV!sg=}a9?|d{!v;l2!M1nG#-Ep*E7T}Ev#;U&q<#k zjh;H{QMa@KH(L=3LXl%El%UTBM|~l`>nKo~tBy%+l9*dwT?s(sa-sUgd+CKXJ!(H$ zY<--;P5d`IkDS&(Jxybw-cC8VA_&IY-($Xh>AF>G3uu7w095{iYx4BW%-0Kw3Lhl+ z@gwem4kv7>3JqYEyS?=Tq7;`F-670dYIcmPkifc+^J{b>BkW=l)G%R_SuHl5&=EfjRk)h*x{h&)&hLAvuZjjKrN0C2T!LA0%b=M2AV8ylRcUXvL19RDb?HCwe>P(2gt)mo8I z@NIKO$8eNKWQNsNcqXf$V^^s1=XvEem-xa&T{N>gQuZCEEjQ_6cXTzNTvYt!oo-Un z)!;jI5^U!K15T=hdU6)g;nTccW19sRKZbXHyDAGL`+^WqY68K_YSJ%-IG8WRBY%>3 z!P$G|-QK9a{ke5u239KbgIte6c0G@NP~hqwcQsN88S(JS_K!1b{Xbu~dET?4%rDJO z814)9UpM=sk^3=Vj`!Ns<5L$V%hj6+4DM5J%C1$fRned9ELtq&U3L3-X(;3h+4E?> z`Afs=4Po)i$sMaX_@La~9fY#Z61pi?ao2-N?^MzulYCoGsU4% zUV+M{q4u@EM1tJ^Zf$?29ycP|i%UMU&g)O<<%D_S8NPZQ^H87i?rb~ff;WR%kJwHR z%vLV7ZrC|=ossy102fa0g6hh)lI!0lTuLeiGfj4;4eqm!-4ns*zThuH-$(3tVq8F0 zxZ@N)D3uNfl1X40C*212Z9DzCIND~>^!9C9CanOgQx)S^=<=NwKQn*2?~$pOjTG=u z`dR<+vg-ivJ#No>c4NY@#W~PHB2HAt>LwLwl0`?po3JehDP3MIhv08;StiB8w^(T< z`ARUAdd8hJxinjl0yf*3Vfuh@faJrrVfEn;)?oBbxoN&X4Rk%z^9;f0}9rMr>6cy zH*+@dtq6m$ub_?3M4YQJM>a{xdeSQcZ&t>eTA=KdrJ*zy1Rg;5ECl;(|7y&V?Wftz zd^N9__EI+&n5+SC#hiu7BA^(kD~Xb-IopcyoDSNIJj+fxw1FQ>b4BVqE%A&8K;4;u z)&Qcjc{0xkF>i_|p1;k8E*V{qlG7zrNyt(OX*f%Nx=M}<-E%g{&tg(vzg|Hr(|0oI z+UUC`u7JqMj&!-WuiN}2+$u|o|xKOGHrcVv9mh5t^9gPR}#XeBJ}1E8ilI^(Ig|IgoXftjp-AcyEe z^6l4^wNf3u!*fuP2IoM8o<1_7b$?%7%1~byRul-(RtSS1$-3{4wFf*F{jBztq7GW^ zzw0`I$j1>I=M5GHen%1-!l>2;U8&1kKVA@>wR5fb4#WW5zSznE&~ucL%f)iGGlvr= z(Kxp-AcoF-m*90=-jAQK4havTi5yGC-*Ke>9sP%eqeB|<1kHALZxra^7LWf5fds@L zeLMSa9P@waNTZ<($3vsTcm8kI!iJ|HrTgsNr6Y3(k4xWUxb8{TbhOqgH0bvK;#cV` zG}yX?U+n(#tQ(xHi)d%HqO!FpwKni>|v~X%pqw}N@XcY!IFFQ za!woZ`6-1N+c7`VN{a&i2#cachtEc5s z(niu*><0?7d+7)x?vNa^=^RO7Sr^IaJIQp)Tag}xVB4LEezt0i#;Gw;|&~kq_vG)g4 zG1x}42Wh8V{X>nEvUQQ9i;3f{O85WHu}?1m{2J$R{r-O*6NJg6`IU2>>CQZ#Qc!2N zVVYrly>D-JF7(5896%d+r!boTkT@Lcbc|arU+T5(y7Pw6(`AnNa&+jZz`29hBjnE- zoJD^fWPWsflRSb5!Z39=D#K#CHtJRGPmO8o`V5JSDanC;)AQpu4t@-8!47loM(0%y3EJ?~HJB#j9j*=W9?1J?AxEs^B(!{9OJOn8B>uwS0-GdgiI_c^dd*3WGD>2un=J9{8{y9H7fLpE5drrFl zTFw*Mu2P0>nFFbcPJc|&CF`d9moxTXb&rE*Y*gEN?(3(7i(~m5XCkJX&k|{ty|{X? zTht4%TY*^;+T*qZ!Hd((wxRP(Vs#AV?yq9&^4|Dl@84IqLw0~Y02F~76-!hfj8DCp zgf$(vrM{<6C}p+b0#IY$%HDAP{hc;3kY>sv;{SZi3WsEdhtwkG!#nk=82+!|lW{u7 zcy;Qmf|Tm;z{USCxh&Q_(IsPxFs3Rhor#M9oJ5*cckV-@tD()QXanUJ)8D!`Hv8`i zu$bsHN=CK6HfC_1>Fx$!`L_*31}^wYP{04NT|KUq#ZoGZ+{{Aohn1}>)d{PKIX4q- z^?XVrLHi!O77+=aEXtA*fabpc`_#Ass&Rqj{L8xuKryE4{+bDS!UktZv|jl@aPEMq zn8E2gc6^OFJ}x+$D1D;YBd!i;ikWXYw+!+Nm-<-kL;Yf|mUbQ-og)_ww!j4UiI&6d z?ok5q0+*k0Lnp7*ML0djh@G%IqW*B&a&0e=pawOx^pOREjqF zOJ6O%59!Y#oMtUzu|wad>z1iv%P(SR%<|E*A%M9pX1r*USeZ}aG%Q&gzP;I&BmD4& zh!sO3(uYX%v+UoXto*U2M$#qp;olD_q=CA@H}5Rn^?<8dD@0y40{-`lfpUO&9D!_q zwd20Y`$cZ)zL-If;e*Gva!aDC*)P2=@la$a)W9Rxgs%UGw?5;MynZu66{P0d1rEh{ zM_);{-fknytLo*ynZzgtQ_gfEze92!Y||7BqajfcU8EMcbpfp}bmq}EyF2YSKc$MI zBzi5!zv@1Wj0exU)e!&(pdC>gj4yvH z9Yd81GEfteit@W7a&pcW9*)iCF$QL0!5tlRj6reE(S%qC^v88v+g#`SI93>@cOIqP zWVFN;aN_%*P0{R0se|tX5dsUUqK_MX?D2jE9(=y}v7PRLG1Q`7@;h0bM2$n&WSJ=B z$u#4?q3`<-M*G!PGA7{F(ODD#uVbc%OXZm}S%i{oi}axI{z)JjaDcavnu)bH==m?Z zsggH_seg@hw$bOxGU#8H*A1+>+$7~}d!HfEY^wnb{IyVR>Cyds!0e_M0I!edbqegw z#{SE)_h>?bqhGgi$k{|bo!*=d^_%`;Mt>lPZ7}8WvW5P?DsG>vQfYwxMZTVVY2;PA zea`!fThbh`i}*o$f<7&wO#^0qwf8dci`8kcM zq~391tcByt=_==#Ug>=;)%*si?aGzX)k7n)x7u2MEQSP;hzT}qUH+PIKV%TF`f7BB zhdZ$ahik2EZ+-TdKz2!^!XJe=D^zK5oGG%K`=Zxc9~Ek%L2HJxI@ai)8wuh=g5nf5k2k!HTUj&fU4p%*$S( zn{>FKz0I2T!|n}b(n>c(?UtM`ecEEdD8pfxNJqR~_gb<6O3Du}ycy|6x$^AhId%y}2pH0N_pZ2Kh zGhRC2q;bgaESZ-*PSXOC#-%g)bze7&&K3lw3Qt5nVO!DNl}4rkQ!a66%3DCRWNkO> zAA5}7@6nr%S7{Ca!=bBw#Bjiq0Jl*NdObO&0T3B@3KBuBw!-NWFGBs07D9IYdb5CE zsM*yIHnu_zLb89deIPm3v?0OTp_eF9l$+~&Ou%<)f-Mdx7aB*LIHxfxPwpjfirCRHr%)W*8_$@rNUmR6VF9Mp8K>UaH5FCmEW*& zGbScl0X`Kh(m7yM#5W+zMXBrp&yF2q1Cs`ao{o;iz~^3r0wi!rZlBW#_oF<~l(WCe z+AMIM_Xbb3svo#E`zPZ}#4X!^?;SS$(jR@O4D~_%I(eWvw1H}Uy(lVtT%2ywn55v{g$ORsBP(&Yh8Yr1UhKqz;?rg@9KW$SR$OArrd3c*ihi10^ z3Wcrifj4uG7MOpm7@1>h1r8VxKpBjplxjLTLn0Ga=FNSPH`M@NlB%CQexT>cLOjvB z6QoU2uQ3mZ;)wjy(_MgPDmNxCs|vdjcfc|}2*#a+RgPnuBe3*^$9p2l6g|4$8k&<; z1MA;I`)aj5Z@lqthGwWA+peN4BQf^D$MInDQmQBWSdXn07}t%iJHJ1)mGZsod=tR^ zG1~O2UpnpxI%c0&COHAf^Syy@e-#J()9#7z{G%OMChW3u5}d1G%?(Z~sn93LhFF&l zbR>VqP)kTawkyW(<=*cLRe{>E6-SF7dxr*&f`q_G#DfnnT6oD5+z8Y9ZluWS#EK*o z*~f%lBj4X=yWL3}siF#>qWw1=NZJw$%?5vPRZdF;Hr-DW3{qAni!kD4n4c2Kfqc~( z0QsUko1}DXt0*p53gMOC=$;gF_d3nydU8jG9FvN6bMiOLEE?`Lb+0S$GBsvf@0}Kp z1>_KZ`x$c-L_@6NB|m?j=pX52tszP6wf&kfdkl52_pTcq+mNU)$FM4$^^3uuU3=R^ z34j-5Pbr;0>N(YJ<3+pWe)X~!PLFXiiT6W>yc9O`@dtki;@Mj&Tn18M{SF_P4!tQz z<`Ba`g%$m9)<{M%h*@_;5NQfb;eZRe(ElQXL>omcaLlE1WD6N)kE*r&9Z>ra1QiwA zEu=r)e7f_3pULit^8X$ng<~&n{mOT00UCKsT+A-~@fd2l&uBqjbm9YKZMi>$`JpX@ z{E>S^?a~88EjFU(*=Mc;4KYavG#D34%kUlHbdj~&42g7=K-Nj6lw`Zh3hMR|Eo(*) zon`($biEHYx=a${Ijsm9=OJ&~o-XVzV`fa89uPI#2d7p@47A$vWd9REJVemRY2N3> zp^vh0^l1T-g&DpVf7tNlMoY-KPe4RyGPh5`7~N z6#ws?LMo6L0b7TOW4IbZ4x1r?jL~}d!q+KgyIwQlbf(oTSsh{z)qRM?6c0E9F|c>q z2P@kbmD&@qddujM6YMSLPFU!}_=x;M)faLVWXEjWFA@qQU2xdfD;ng$*_rMj9yu^w z`&^IcFu+^OotR_P+;>BOMViQ-%eJ!L<$B3$3)r>gQkg(pChgv*{&&svi=1a^XkGP^ zLDCqCve#-*rj=MT-}h(W;OqPQ5R@%2oz@=sHq=Me(ljT{hR*!%Yip;-r!j16qv}kA z0&eCO`->|A7!cZd4d<9ZbCN+`rAlXRQE4u_V77u(eQR1=A$o+?C9~$Stj^As!n6HV zN&}h@Khfnp`Z=6s;>LNUH_4d^U9sX8T-C)ck1EI7${uV*W`7Sj?cP&TlvEV3$FrcX z#p0R6w8X(HOgms3Khh%DT}1bFnc8MDh#*XhVIX8CJ1I-Yw(AOXalNE}dY=ojV9%r> z6wgv%3s~J=IcDH$=K+N)Nytc9Kn38yC=jB6-}ilA;3doI5Z1`*$ZTo$?(abp*Z?{I z*#o{mpm!4Fq_AC6{{*R(ueX|{imKZf6K@XJQ|<>U$*9WH`(!;__dA$Fk}%UCTkGx8 zk9V>SH7mwYH@^m*;{U*Cc7mf~&x=LKgK$^?U4eB>iOJ(qY6`9VWA8`M4GN(kQy&WJJ zXs{z#{>EbaJ3R;76LWa%78}PI5F(%SPZ#z%X(<71pimxY4*{c$edaq*9?@0ssfK*F z!h6GvErH-0Ewd%O*W*mo^>TXWLdGh}k^^5GIPbst1H=cX^~8_z!8m}t#Ih}I1~?@4 zN!fb6v*_VAOztuHj0&83Z1eIviW#xX5;o9iT<3>Vz8aD#b>X6CF{-ZV-L5SU4 zDMW|*D2fdnaI32mh75^+gNb&?h`(@&%{%Ig@pu6Dq;E~igPtYw>CFbE61D z!1_F|Pwi2zRt0jWzRVwk+$^);-rT)i20?QTo2TuB(>GUJvgiW>+JwpNh5;x{=r_}r zC0e(!_bIqK_;5Ub{BbGI0m1)gYjGq0`&34?w>*`MLbsbm+ms?7c&4 z*FZ3y4>H_m!h;j1;qf}jH@^?)OtL{-&)~1 z?w$)g4#5n@?l(n`o$0g3TV*mR;XAirkxwb3Mk~9u%OA)sE#24}SC#SD`(^Dp>?~{n z8PBw%_7d&|sHZ<%{kovB7z8C2oZkjCk2nhGe~yBw2;>rd>br4tckL8lcMVS6R9Mzr z)xwjbp9O(K^m!8BV|SKAw9r5Izm!q8QZ?{sz*aj9Kt)}7W%iURsw_uCcX*vPY7Bi z?SsJ-I)KU=yjw^0CTan;(S{j(XzS@(Ty-+?DX=DO)m{#YfixfsqxWnCO>sdE4}Ajo zvQ*pL&{_O%9(TJA@JBnpkXBr-*gWwxA*MfSiKJupc%{}jn4e*by<~#hO1f}Gl+35PwO1wFbh0LI*CoJ-wgAu_x38R;Al|`>hjm`dr zeU|~6GyU|}-?S9;7Jg&X*7-!iY7GAmS6HaLxWh*>Ru*pGsk7l+ZwcWN2G*MkDszE5 z+8S~cM4H+&7nlKO!5x8SiTUnIcO>vNZH75~niHxiH49gY8hF&kDAZS=^DzruHT%V; zb$!~0{Qay~=cQ~h%c&uuLDt|o`KP8YudJ2+OB{3nBgK_tn4zkIE{1+ z+Dx=nD5?+G(68B;vNx%?jOix+>ArO<*GA2~&Qf>l`))#@ZLZCFwoBEr-p1H+$$W)m zWxPuz)6)4W^yP&Jo)+t+@$fHSk?YKax@HS|U>_dh(&J+qcaKWaKNi`2y8W(UE1C{r z(VZ>YnZ*yf_Q8Iry)X($=Mo5EyTq2?2_6HJ_Qf_j>oK8b*ov+wcn>8D2z5`1sx}^# zH5#}n7q<6q6Q-|nWledpnenkavE}1h%~IZsSo=w3i!l$D_dR4r?Gj`lcX1H=uaBG$-&A7~S>VuTB_BnboyS$kzLIcjhgzQsdw~B6GrTbuSqA<8y@l&GY zcX8Qt@oC*lRz3LMOJC-Vi>nH3m!1;X+G|7!aQDA)a*I0ebbi?Ly*)E>;Nj2tv;dBr zHtj9Tb@6p9iN?fnW6Z^kF69OnNVstjGbW&7o93uShoe|wp`QJvR)&OgKuKhSpf9|! zUJ=xv5DVT1x!0}oB6f1UK+ZW4cO7dQ4I;m@&i562rU3#5p^HjeUE8-g#+_Eaaj)WQ zEoL0xKr|r4T)cXeGvIt!g7#HxsWbsw^MrN`2>`5oT=A!6QdpCS2E>zYA6SVr>uZ>p z65UtcjA|cX&H(Zz(ZQ`dzy$ZH*rf(v%y2QHLtVtJ^Y{t!hJlzjgf*QPSVMx1ZI$TI zJuEDr(`ZewE&vr)TL~dUHf=USM+E6SA`!dJR5iV5$14bhU8nZBSLfBQm`QoD z3N>MX3ireF_sr87G!@a-c%=@;kt_f&4n+uK8nKZT!hN;Rse2mEjf7<|H*2Zoi`(oK z+xSp0vl2PQVty^$pT(lMaLF+mJh_b=|FQZ-_D5WkBhQxas|)=Dd~zV6gYz{*F~8 z%A?W=b63vt!d`rv!&^BIoWgDkSsj%+@_g>H&+%S{KNKMrpvm(CSS}qr{@7c(uogl7 zC)>y>K0o8LsAd5c@a(d7iEMokVuYj~HJtDj9YKU#J)^q70xAwGDr{1Q08W2M_WEe8;d2gzt&hE{Rt;8W<;HhX;hB5dc4 zjeN7h>(B>6gQZN=m@P?&pS5|eH8CLE_)TCLb1F=J9dRq7-##AgJhu>6{uOuy)=q1< z-kJyn**M3as{rO9_n;&!N{_w_eFs$xR{Wo2MEgxBOB%$VU|{2M^CJ^L`p%41@$s6H zwi(k9X%f~!P`w62;@^e8$BsPb?MHuP*+30FLU_hAmXP?mI*odst>f^t3T*ULw)C>W z{zg^CQ3SGl^^ysc2iYY`TENHj<%%Ikwq7VjL%mHR)E*2_dyqx3wTu=dh%SFzww|i? z`&j`A51n=7W~(a1^YqZUJ)Y3mBj3Tn=@5h9eEbeK$l&ainB*G48om~1N*28U56nS< zrw_Hw(G)KqfDmG6tF=oEfm<{8+a!C2UIQeV;z5$h>jG$?1-(4_-Qyxu>bc)Xy_@Dx z_!QS$!)K0NYcV2jCjk^N)O*pIjUa((A$Q%+Um#H{T;lFQLc2>AsNEQ+XZvu?zMNP_ zzp=^@*~hN)~_BwM$uQyC1ISTjl`M@6}~c&^Gt-@9Xa1P!1DQ3K-hHqGSjUi=VZ+e zqV4ZFga(G#YQ1@GP2c7YQkB`ZU0kqk>_u4}DkxE^EUVj7z`SQV{%1v%>2IvXOFq9h zTa4Ch{sdwnNy1$R0@ve;Q(+qun$ITBYjt`PkwJVa1Nc;8^9K7-K8=QZ7jUOa-mG!C zZI-6RdlKeDV=A03`Qy)HI8%s^VEsM+^k(_J1XwD_tqX|TSRAn+sJlQb*xc~7RwKp+ z#oWy}-yec%0a0FcDZr-_w@SswGdWrr2?ekn)Al6n795YQN8_ku!x@1$tEA#W$8$(K zN^S!Gbjyzrk4GxV3RGVgfOxEh(4Zq$L+)rk3vm-?wBS-%sD1z!Vco#l%C+SH@^4&m z9;Z)Wm7j%RBzL+!CO0`?awcYA*GKPnAE}^$f5%!J4ZEAr|E;3GQw1=OK*6xAINwS7 z#C!7d;DQc8v@-C(%Vu4Ij;z@f4#KYUIQLv(oy2)mUCUW<8FH|VV75R%Yt2vAc;KL? zA&@M0m*D&!*Y4{QXGkCngZ+Hrih#ix3Gi`s$XOT@M`}I4RZp+I&EK+ zyxBFtjR1~qg#nu6~D2zTrSQ;KHM90px z8oUJl(oNiRoO%1m{dup_Q@8|vZKdn_o`1&XnG(E=FF``jlL-6+dYH(OTn(Za7vP-` z^y^3iG!T>wgHR|`S&V||6W9hfkJ*Etv`wyIelav9Xz=BB7jRa^Yk8{&nHlJ{8aIG3 zyc%&Cw@Vr9dSx2_E58X8#M`LIec$HlhQiEdfF@WA{%$=&TKFKO?gLZO4d=mm`5+pp z6H0FfjG;E!-6CsIVB1fTeF>2GDgoUI9pyiL9W4r!{TpicHXsWR*s+o!ScOtk8?mi@ zReD+K#(^YQvjPG&*%2O1-f0s8xJ)NH?iolMHrETCzdI3aene%TCen1=zS-^sR(y^o z@6Kbdxg!e(UK;?!i&B`2oz9!NV?7=2`8VtbM zqfZkcc&4}e3`drMNDb9ZHB=B}u7F+BK83YinRGIaK^7vB0w9t6AGIS$RBa)!>#2-$ zVp#V$0J44VL9Bz&wVg`5a3an3A^4Fo`lNF}WNIq`oY1nx4&VAmB9O;i%-G9gitOPt zW@izsXBA{sFAhvW4C4rxD8@D9&)ypMRWyYb%89($KHt~Li2|TvR1kJQ57M31I$ci~ z+XWFF*B|h=r9Jq*Y2X390ZLfJgjF4Rf#pj$FHxdj&pY*&DWOmzaF8I4^f(<4W=;g@ zX$Q~7#X*L|2H8wXuvvZx-QqQ3JZjF!NC9so%DT|B+ac(%NrftUa*Zrqm5W}2Tp&=< zb@z*kQ=MY3flcy|f%WI}ppPW$5Enso+ERKd7Tnzfev59fHvu-{9Y98J)R{wv>kIrH z&*cKWe=dt<5!~CnsV&oRu$%I_sRPhUoOXBc0fO8pnM;pF-N!e5CJn$fwab9NQ!VCH zAW1v0yMabDHLveZXXh`WL*T^F0oNH5FWthevsgPorM_DGr#lnqF}_H7rUgLOk2kO!GK%GVm$ zSmSa)*DQ5(_VV4)9?K5}$yCjF{FW-sRenJv5lHYvE4Q~XaQ zS{A~2BlNyvH37ha9b4UW6Nvkp>M}l}=x0!9si6y5`BWxgs=CtS{QeT=1aL4?wp6`W zZjrX`IooiQ5iE{Au@yk&KnX7mqKtO31i%;dz!$%{WWPA_#YEtv#Pr&jF=@yER_e~v zekN-wi`XV-(~KbOpm*QX`Pxv0=+kFr{9&jQP`@eEO(?6Ye>buoUTd|O3>m!Tryyrf|3K%)!Hb*^{ z?ebhH`9?(3&H^M>-Z{$jh>DU1DvES4oe+zX=E3rUg?u1 z+~DdiTlYS~zt0tjD*!X!3xG`NV7yq?z@2j#<86FMSIx$#!Hf{N&ZYdbuFJDtQhRAX zvD*weq_6WtSr6s!1j3CZZ?;oxwOo)mIs=|h6<)S}uOC-U%~TLp?cCfjcmu5Rk1t?; zpUIy`kw{_!o^>l&I+_7)>VPO+j|KLUq-#WLm8M+_p+(1b^EG~?wN;V;lu7}4vnL>L zhV$=k4}1$K^ZuT+o3bE&0$bJd@sF%!05oQLyH|&=1E4N+;vMRLRKhw930rjYNq}X* zQPBbRAV&}KMUQjXaQPJhc@?V|^HBV#OGP0S<_^Kn|=YS z-Ye^~d-)RI1ZV}mm-GR z|F}p+nx@9V{K^|eoXoHjP*S2EN=o21KZScH_}_#40?Vvi{-yS*euc-iBv_FuKdIED zAkNxfD4ZQ~3plo*I#u>J8K69)P{tH+)SRUs(3w2A2v5#*1zA!IppsNJxGgcR1bC@z z+72l8!xL? z045dz5A=jowlzo+X4uNhGU7qi7NY3D#Y_(qI*L5PKy*AarO9VWWZn zz~GLW#F}N!t6_rW{Q>!VD&k4P^$+S|_GnoHGP&_l0EV9<1D~$g(x1U;)_{Y0g>^O6 z$M85{+5TnQp=0BLf+|)iQ4oLdk+S?I{N#B9q|inYbpq`$O<_M!wfqrRLH7K_I13Ex zCI?WUBdy*1|Aaf>?C9sTUJoGBzdv&2+{1@q#jq0SSAxqCtf1cer92dAfr7E2*WuRp zwq(HyIH44R161c#h>|FXRZN}SP}ZfBq5)0gS6U|3A-z;6_3 z`w8o{G6N-e^mq!gK{%5SfYzYROTOIF{;-@pLbpRDmg z--4L2g-9VOj&7FXPgm9tIb{e>hY${IYTy9-5Dh`WA3_)?u$Zuy-wwDMI4{CPejZdH zLiM^m*O6lz>A4?uWs@|1a$mJFO`h8ln}Lx1bDTU%bkmc++W2J_+ZQh|z)?&9X3JE( zjXvbRho>2+x6XLi~npcm?&+*JQ*9)K_UT}Ju zx%!sizf1!T{WJKK;f+{2Y7X64-WPCEY?JRyU$MbGD^qKjXyu>h+_C+iV~@HI{&=zL z+&&N9tbm(P0evYWt26ykp5m$F0otG9zFA}mPyVSG@Jzg&4h4WKguPq$gpkU^7MPhu zW7@&_yegHtp0l3-M$1G?&=Y$-TY{!Kt-hyV#3cOJS;KNNxtHkmaWacw>4hLWqR_ny zH~+JsmxmGf97R*%4!??`5o~ySS6-g4sJ#jTjP1zKZov^wc-nDUq+JKLLflwWQ$NN6 z3nbLE!qGwYBx?9e12W!&GUAyvvj5@`R0Z4{lUV3Gs=bH8j%%ZOoI8RBH$25H1;CY_ zQS$WVt+!$ydP{*4t(JodS%ZM1V|@%Jt)+v-)kLbEDXoZsa^TU9h;1hO#Kx!=d^2=!g#B6dE0cY z9}aY=-i6dkvmR=RGU5}lHv1apOVoT8fFTeu06FVUrU$M6Pw<4L!4G=@9I%6_0%BJ( zI!I|Sk3#F)ZJ_ZqX@371w@QC8z2nS8QY`M;V4^}?rGt;(%Z`4W3oY+ zS0R8i(cOcY4hKAW5@t)D{xCC?dJGB&EBcnVNo6E!L5;AwVdHOfyg`w4rvEefP)h-SETv7mwk#*k++=Yw_L= zc=lf-rq7M2=m8Vabp!0AwTfQwP`v)i+2J4H^w@y28|{es5C3KF#QrTE?5`DgTh6~P z=l}Bsj)pwY?uf9_23l0!Am9@JF=u6PZDZYP7jXI`M=nK5W@CREYa z0;bab@D&@S1rvC(J7U_remgq+!0gUrcE3{Ze0@s2{e#W7Hw{_q>b+)C%Vko%-7bUA zdHG2d)(C`z-{k)BWq6b8NU*Ap-s2~kzEcR{a1 zXzujV3D8kA0lfZ<=U^7x2BGQ&aQWFt4C;OadSF~JaxHMsKtQ)`$!pf-9J+ik zYd)a8&XC*rUdQvm6yY>Keawy#TV7gqpHj5<`2Jm_ z4Qg~};x-l$y6Q*O4c+=x_u1_10iC$Ot`l4r})ypU}CYOsmBW+JS7Y z&V6_7yBkIS@|qdAIzMk>i}eTGAYb;C-2MqJ^5%j6Ko_gst)7j(VK9r6wNmQ|PqBKq zc%1&8-P<5|q?O@MQ`Xw2ro8DvWCL3t|D)hB_hfkMXr!&%4C5=>= zA9ZP7C*6X&1Xw-N!z3Nt!hD8&uvoSq?X^#e7Kq%7D zU_Ot{%5=BBk+ZId^|04k)t?V%?VXY?rEmgOMqrsxd%nMif%7l=p8lrR#&{`)-;T9G zu>Mg}V_#iJ628}@47b~7W#sE&uF@Y*#AYcX!zE*#grAc$e`%^}74@yb%*ow`lGiw0 zvOlg%Zhq~lUM-?92a`@WpYekgN+8^j0Av=lABvqwplhTcS!uSAaBR0s*z8Ih-$=VA zHdGM$gtG7oR0RTJMfqdq*fP1c1k6D2V!!lM0j9;$i`@x z$!15C-T>yj{Ue!r@i|Zd`#$mh!1W*I))_U@f;piWIBTVm6i!R37| zcHew_+MwisDxgzQhEUOAz~6oXacKZGoQuk-TJ2xIF$ub8@VhIUQ6}1h=c7!#-I}jC z0qhZtal0;QgrW3a3`?kcN0qN}onh2v(7DH8ij8c#6f_tao=6A%Y?#c{N`%eYLZB9u zX4tcrU3!12xt4dffR{=5x=N514{v@ef%-pCyHCFDj^dikO2b4d=DBLgo|{PU@eXO2 z`fwZcr3Kv)?JZ1sVe|T+p(i--j|4NG#f6NI==CW@+g4Mwkf2rs_uS4^WkQ#41Wn#` znNFWyF)l;~XKM5p6!6qtN?UT$w2fGk^ImOLVr!t9kIFVIX&{|q5-R)rC4&7-6%{FO z-HAR!m9;40D(&Z)i2?VwRjHdjxYrov+y)_pVKdnnK5@M{& zSuS2qzj|Ytm?uZa!NJg}L{42O5`Tbq)2W&=30K#+gKH&V)p_Nz{A~OBD`GyjPs&d1 z_N+3Vsz`a~>U(U)@8!z^>sF@U0?~w8hDfhK2SZQ2(YX6z!8m|*t3v`(DnJs0Q+;*u zWUvPOiFb$s3L4gG*Xi7whIh!R)#EHMP?L=wpgK#qIXI}C+NE}CvRtl645Z0e6vi0| zm$|x6^d%YTa!KTWA5EMtEP4)qianA&a->gzIe9N=WqHM8)}eEFC(6g@62 zp1>hFPwPviHG6UD8R!Hyv7xQ`(oI=Qe4(JTqhRXxfU)Ot615i?n>yjc_i0lre{-|k zd1izQ-p|Lqy<&oKMsM>5yQ%Ez^elq0(Gjb`$#OkjX1_o?*KK^vAAHx+kCzusk*9vm zV~T8gj6pLFqu`+RfWi1b&zS2jIQ0A&15=1W2cBp(%I5|yyKTF(Z6jBtdMG;@48}JZ zx!n}`CC^mWbJ{HcMIiWX5;?fI62}VJUW^PU35<&+dfJ z%ZhGcxjzjfw@o&_=r1aP&R8L@6a!YLOm@di$h=5O%q6ye;!muwv^}kD=Pr1r?cOF| zzV_0!(tMZA2y_SQ7bbmCVO;++=?qoGIYP&Qc<;^5geyH^P=^|~nN`r~cWY>^I=@44 zz*8lAM)w-%7_asEQTN#OlL`v<9CNoObF z67S?EBf_3hJqFp}fgY|80SP$E!aPPhW_sU2J2|#h&*7~h@{2|53ZOrc)lluS=B4Mo zKeO$emps7uY)E^ac;X(hPMw2w)N7A_Cc5Pi<|nzNaAYx5Ydd6Tiod3Q2b@oNWd zj;V}^{Gz?T&sK@g8|ILaCYjM+TTL}HM)OMZE$eC&qMro#Z2j8;jD7JhKfq-sLyG=< zMhyIjT&E5e*)vt+0=fRVx>K`+ig$WCT+@1)Q@Q*EZu$IqN}cTGPCCNKznmt++Mh#> z3iC+A_gcV3ewOQQ>T`jeCtbHK{^ql%`Su{F2CfGkip*m^ldEi>=0!R)lwJ8dRukJX zeP#)z^?K#JH$~>fe>()PZ&+lEKZ+i{Wx#JlxTUWsB@T8pv$@nEmR!oYPK{Zhj$24> z7{ctX1t;8&{C#&qpw71LELSP2;Dwp;ABox9_)TkVO%7+43KDi^Ct1_q_IW&yvsWkK zbx;jF0j93%4n3=+GV%dku+q#iQwh7Q;+NL6M$o#uD2jkb$>4gKx;qv)sh- zIhk@qd~2i4$Gc?EMVY%MkwE5oN2}cJa3M93LDYPJ*b5!S`siO~5yOI<0(p;0L3`^` z!q*9m+@Em#EhLs+$FXiyqSM62SmWHXG_wTl26{1$KY?{|xpF7m_gb>fM-=|A2^?YR zgpZ=Ry^xEzWbP61>pH$$QO24%rZs_l*~f)QVk+=yb+Xa)lHKJTFkq zH(INVA9+=$1TG4&-i$Fze}sK$?CL8YNl{yc4t46jV*SQ@L`^HecVA#_ zCkoGv@iL~32c2TZ1pa}Fgb2?sq019DG6;NT$b8nw2$LmY<|R_|esbp*=-0G%jHnyt z--|j5ZnQklPe#!db*{|DUHXN;)f%eePZvKYQh&=>ecPQMLufPI8hw8nblmi+wy-D* zXSifW(#0~%+9mN^meBQ&`5!$C;zdqgI!?X2EY<H)egp zQrhrGK0GHKR904gMXP^3L*0u_(vu{!wEt84cdeZis@{*lK5sk8P`#p-xaAg3(xnXi z0IgFM65ACuTNPr(U0eA}aOzUxMa6}4o3C6l+A5s=79U)W*Qnwpw@YV0X( z{!r67r8x9|RFQVwq#?VFRFoWv?*Z;u0tT1Tr~FgHNDXL;CtxqLNzb!NJ^9i==ao8= z$QQEEyK5jM01a(<##uLke@fBWH^y5><+S0J6pPsrizLz6PQ4b#t4HOw5MV;ZN9?^; zF+5o5<|wB7<(xQXqm8>hOgq`#Y&Ru(a(vF-$3xbz9~y7V-c5l2D|xDyrOsqG|HbO} zq19r85>$^tqLj7<{X5r&fwGjYjzqcqXYYYC{bj9kLcGoWTY=fzCzY=kc+S6l^y3tD zbUa&X^;DK*@%HD0ZN+|0J*Ua?;Gj7&im|(TjZWFLk)9qS#0eBjP4xx5>PAaNATpo+4 z3;G&uv{VH;g@Fd>yW~}qa#n1~y%WQ`bHg2)^f%>Vr;?i@IRljMHZM|2dikmQuB&?G z!}YCR*3n*2{CdHNf)SgvXD)g{KteZRwJs0k5=+Q-g$FX6e01}00k^B3&m`8{KlvrzyD?Da}BFnaowoj>RFj3<(4>k^=-OHhAnNoC6xDaP55z`u{clQx8=1lT*6E3 zm5W-Egq6?;qbAuCYBk})4}U@#Cmx)pUlvBkwD_fG8A)zng@;o293hsob--t&y-vI6 z6KCO{Z&vp4jO6ikLv5q13?uTb}bf>$}b zpJcsLBnWo&xdvT4yzVX4Brk2K3po6-Y(ZWbfLa{67Xq9C9ncvSm7at+@ zbbJ&mx?JZ*ke7UC+FN%imkq!8nS>=e$L*2S`!7lFH#_2M{>HoV3%%qI2P1=L%}Tc2 z4a%{C7=I9{-Dxu8c(jR7yVZ@^Zo`NfacR-MlK`Hznzh3A12$?eZ_wf{H4XUz5H=C5 zRY<7blM~x6TD3IW%`HK={Vb79(?bC@1rRIMd8viNwAnNl&vOzTi769~_e7`i$`k9P zHKy8T4fuqInQO{IRx=DHW1b+t{7QB7MrsMm<0o&5&XY#&Ug6Taz|tLp&png%6GUC+ zXi2=#cWjrh;I{#^huE=`9^`I&77L)TI|&o5UiXhyJ2kvb-4 zc6u=2Z6Iwj&@6S&g-m87h(2xPd+esrm)vmmp{dzR|wsh2mw^(i9Y8dXmxYm+|*an`34FS?5thf|d* z>iw3w8lcvEjQPmHhQ%oY-3Yp=?g(P{w5V2z_Ilcry#hSMA;Jag*T>mB-O z8wwbGL6@{Vr!@u_(Zxpk&I_OutLhoH=7?H9?t;88HlVHjsm&}F!AK!B?{eB9a=D4| zh-Lo`;NH5r2M>&KxlS#8Xj~d`P#B0-7M(byo2l+9T&}AEk9jgLa@z{YLkeW#vZ&Kr zR@QXOA3y~3e-Ut7MQy${@%ZR$Ib=dwlLED6KZV_gvC-0r_|!guC%Ac+p*6xsStYk^ zeTcN?mZE<2gv44|%IOCEmlO-HUHtsRJ1=;FW_GyOKHaGC_7br-_EUC1a*hatHvdZ& z0S_5cIwUkpK`+@QFX-e+$G@&led%>1nqS_Y&2U`0M>b|W%*RZkYjL8Oz_PIR{Gzh| zUx@ewf(fsVTV1;sbfAC47KK;;j=H`91c{OHaf=@*vLo=@{Yi9kPneB8&QY&_hS*r8 z-kH3<@iCg5UEFiv*OjO*DIe&sEckg+mUk5FP^%?Ch=EIP$Q2OEr=^WFxH6nT`ffZK z;{7vn)?{6}pA*mFyAURB1z5R7yU#Y z2->gb_7Y`OgZ`a@KE?WOvnM6TSk;j!S(O}Zvmh$Edk%xv`cXdJdrnErp#7+8I|MkaqzcGiOv=M+q1Lur^%l3ErPbUAE|$XIYRkkz(dj>o$g>~ zFut@A>|(AX+Z?G!MWNk3sx2^H#7fM9f5kE}HgY^SMyt{{b92~ghpGiA} zCI-viOipr7!3B;y-#VH7>umMhs6yqPc2L3YY&lU9P7yG`-bz@*v{KF=G;d8y>uU4P3;wnDD0G=ooLJUu?UTH>{&mSnuYs3ezTru`3nJ-0_VMlD zrRmJ1wAl|RlT!LT5=DvnFXmU4q6@3^e`Xc5Dy7Zv5e@@jcqg)?@-6@y^H&x7Yi!=~ zpECAAlIThhOanB^y+3|YtG-MrB5e>Uy&Wj@ysOM2SE1p9F*14Pqw&dbmsa}#Sx1Ql zfA!?cW-n1)wd)rjR`+*;Zn8DYIti>)+PZkQq!c@M_3~dhR5`H4=d^9nmA_MB47F3A z4`tb&wUC^_P+kKqT03`h-Mcf9aoR-bkI*BYw|>1t-ksA{$jEErst}vAQV;lg+UN}* z#OfK*SbY<<68f{ZvK$e$T5&rbHJF01KT|#pB%TR?2wqy0g*>vpdb7F4!&+L; zWPlt{qU%CxSUbMgDA8h&6cb-+7$e86n|#KI&Hc4H>1@L=t*kF(QqmIJZRn~Uuy0{c zsM@=0CmH}4;EJ(vYwk2OD%{~#N{o~s*%2}Tb^{q`PU=*GvTPqFd>Dg zOHg+20GLxHX|#yVyaF0*&({E1(;xQvKkZ$4IF$R}F6AVIEJX_u)!4@5lq`c#cCsDH zR;EHKi3r1_jwXYoY<y$yOYxh?q{yjHOJ5B5TIhSQ|5p8HV@!bWS?&`@7!v|M$A) zzo+ZcbMbw?_w#%{_kDjq_buemw5d8ql5hvfW3{&*`$KV@9&AyU;8f?Hu(nxYR1G=Y zV-D*~L=rT%PcEQTN&VVIW{5BdW*PMl0l#9Du1lM(sM+{?;ZjkI*Q=eHpe z8{#pl%yK}IN7gz=EGTLMCt9>wu-cnUtmd34;oF|hhn{dDSSt)|yRGw#GO_n~qwki) z`)sNKCHYI7qAhV5VIE?*HN;oNzeQTWZfsk*R+>Fl(I?}hK9bOF)`iDb;LZsdVj7oE zuon0J{bxTW!EAAd9Uy*#S}~-Na`7d}L>;g3D5vz|B;LEAe}SZ9cOikZ5pd~?(eo8K z8ZR3>&_rb%^l$3Zu?3mnP8SJB72)+Jxs;YCd6blaLx9(5DbIg=(p{Pw zsVe^%#k?}(Q}oiJ39hI0tC&K%OYiz5N^Bznnt+|;a@6P61)OdsV3`c|?mrEWXp5(k z_K5~rL|=s#kIwx53XaveCzjRPnIGhNl-p$pC|DfAG3ZQul+T}UoA%}`Jxi}uC%Kb% zErfJ*zX))8g*hd>mMxykt)L4lU`2WKW4sW5%JJIv@7wG6kyJNnp$^a7oi|=Rzy3p4 z14l*fW#1}Ws4$9}GGzA#V{;n|+~iu!&=_t3CesA>hkkioQ2ryUm=T?AR@z@&9>^%C z!L59o_R``N77C^u761?HqIYFki}27xcz5W8Op{v|Q!64hQt()}5pTN6@9Gn;dovP7 z47**)i9K~ukp?wDi+PSnlW1YPX6wYXd7qUDknP-FK%;XB@M7n?E3L?WcvfdQ)mo*D znV{XeFIyva^OdyQ!Q42MUbw$^)iXV1mvz_Pe2kKAkNVi|PN)5LzQ%UD4YnPf3#h!L~*Vf_9p6`zS z4r{H_7K>$#@?ST5n@PsM!&^7^N6zfz6odiUz`q5Q)b}WBJXp?UaSJ4wOEM2O@1<>T zDKLHLpwjI_p<&32_)>slZ?w{%jH9WQuJ?^a*6&%Ur-Zz4`Kjz`>BhKGcQ3!Tp93-< zvB;p_#IU{xND4+I4|~SCrXr%$@ig>oEyTZ}jIQ)`5oS}%@(+^wewRI$CVzAgjJUTO z%JVt!tldsuW3-w>O|NsFlzK#?xxV8pH?OZ2W?}AMLFPXXY7TtRw(hpBH8Y%92gHq( z6DdXpP_;7QYVJzYV&@pk=VXg+1-^s+$Yq} z?e(AsCI1Fj_pJVvC}n=+ES_8L$xi5T@vKaW8VW%>cJaJHHN5fM*S8&dA$s2T_5{h% zpT8fY)l5EXF~R_lxi`RgcC2IEDwrfs$9==;J1eW4Of}*2O~KNeY#sVm_eLS?#qk% za6>+2MP7I6u@)$z2=7!cC-M@t+peL0{{RuknQ8VKsRtjMBPF0fuW!b2R&BsE#eS4q>O_Dpj z*GL!Xg4c?@R`PrKS3?vh#Eqry8J5wjAN4ifzCrg@VYO11#eben3)x&Xv~wooWgTxS zFo$DY=r*{ED5bMy=(qC`8NwhT0O$TYeXvUeP%S*iEFhjnbZ*aN)UWI{7~JSzc~}H> zD#_s3(x1OZ?_YuhVgJEtujGbv!k#l?4ey`$q4dTBnViL6bZyWvlna4W^MbEtRui_PUQ+|5_rPZBQ8k=1Y&rCHD+0&-;%xq5*aG z94QQ%(AM^xWfnxdb{z^WS@NiS_5K32KIM)bZ~)yrlplcf)FD~L_(!7e;4iB-6edIZ zVwB1I4FuA4cc7FO-HqGAU8w4?caAnmWz#=wret`p-NQTJpO>CIEMc1E$k}Veu~ao0 zRurc%9k+>i8E{H(Z5VPccRSdHT3;>Wc^kNnt~{argz&$S!RizcMr7{Y_Awjw7#^)DD5QC2SdOp`vsIQke>MM`R1YI&sVUMHK1es+k z+_E5w%8FFpb0H^U9G#idJQ{GANPl_$^W9ENO;2Cxf&&_-Jo<#vDGdwNjNPoyjyoMP zUua~%(hHOTyFt8(yzzRui`x&Lx1bzVeA{U5-yr7cni*G039~*_o*ioQKHas@04wl_ z0C*!k*H2BBol+~o(Fzi8&)fxd}b z^uZP{8hxlR+8#q1#`>!~&3g+`YvCA~V}}2VQzrOTfe=&x;~9HRNc-5<%r{ z5bJwb>!pbIi;S|C;{`ZADD)A9dgLvDiPG=9j*HSHEX|@@sjP!bjpl}q`(g552>xRV_&gjyn!+}Ju&g3Cj7+e z1JeqQf#v7sCBo5zBJB!G4pmLI7bVrSh~o1ovknUQq&HebEe$DPK2>DDKyzk&D&6E^ z=}m!}3E9_ineC#^LDdUqxJ3oo+pO~&RNEC6yU4R=N}S%|_2W1L^;tR_oQ7*JPD|?# zuX{){Q*$j9#Z%Cu+R2ho%S|m}n!pkY3oy#NpPBQ@1+3(aVfMcX@0tqFS?nG2oQZJl z{j3Y(NvsL*Xz(j||4LwDARRe>o1#n4=n0X+xW*4=1J<_w-VU=W9I5; z@P8?$xSr30Qm(H+C=(BHM+u{v3R=9NEPDzm?^@e<)u_3qYAFZd?k|68rGI8_@`It~ zoNjbfvuT)LnX5&SKWDL;WqKd9EW|ifl0@X~%W|+2DH!WXzzFr-)*a1YO{)nr>eE}7 z!W>TJNO!milwQ}45LPwW37&PTpGf5rrUX+of#Ee#vLi0#!q?OvNmanxgkNn zrXQ0l-npIBgB4blT!jid9=*y>@iCp^30M!5PmdvOXcuZsPq|%Yne;$;{l?EL<=es$j0e3j0(zWT}hQXr# z%>{8SYBTD9`eo0&Qz6sqK&)iw?bnkA1}__Iq6D3nX7>FCk7mTzeDdy=)%F97Rgi^V zD9gmK#0Az!JGEvSm~7}pt!klT=JwsXk|)A{=U5k3up^QB50b2-G`&lJLa@#D>zpv` zMJc5+rL}Y1*!x~F`CluZdL8#MZi*gu^{)Tc$L+LL>0jUpvYEj+f(ga zxPZUdHenwI5`)t=jZH?5flhk`oIkQ(pq^R*iV0W1kR~RNHK`L$Lu4h_lnJA zjj#q^b8i$+>aq+W6e!I9i5nZ^G>mH9?U-6%Hvb@w8ZY9nu`j4LzlpAU#DmvJT|BoqQ2RV89?2ao;;oy1?dVD zb`5Z2jEb1KlV^;_z6VuvMp0RW{dDU%VJyY zA7uIZvnrR~ALz`N)5^Ov^`dhgr~JNI5)&D=e5-DyqNvra>m^0LUNG3%jL(SZoG?e9u7tSdjThf^CXA z{uq+|l5a-<+&CJ@xbPOpMy+Z^U0pT(bEQW%wLSN0@h0{ChedbmwpSb9w_UR!EMdM+fM;M?$c(iZWd zZh6bc)L2A+Dk7lQ!F;ET_0cOx)q`TbGERb|ZPr<9$#NhJr9y~=uL>b4iorf=HP>h8 z9!%4)RLCUGsnU{qsWPq}n-7o^7kyC5lhz75h~cQp){TEDeO8#*H5xNF!!x9ajSV^G z40RS8cwpzlV~3wktZ1A`DYFbp!y$$lqG57z+hzxS%FrWfOpw2au2JjHW)L{vn$bCV zXsBM(nT5*J)}1&F(+wVidq)r3wiyn|QNGHUo;V1PJ}ohkFyDP41%g>9GLQ(r!YeaE zaSYzqm&m$A-`XGZNyNkGL`LZ$$cy#kppl}42!Tv(PximXQR9jR$Mq?ZCDt}9DF^94 zleS#(wbm!sgTSg`U>d0Xnio$CY$?==E(q`G>3>Q2JmlroyBObVW2kTC`@+8ka56i? zxS_zPR;atI144XgVl5EW3xY{OSZUons6D>mx$K;4w6_-<;7+O2}|C+gT-BEf==4hgn@J^^M?z8Ao`Ur2OpYQqHUX&nORJ!Ha( z30h&ahSdd+ogqxb+&FPWC8)>%WO$(b-yHH^YEZoUiQyfDphw^a#po^{C3;K`ou;X7 zWd;#9nA?0vFYRJDdo&G^cYvTO*XMZGk>v6C94oU{L&7y|P@eQ#V4t=9S&ab8xA{i?R>E#>-?)UHcU^cSGi;*9t=ssY(6QPYgz-ZV#{< zKB{XbWKkY#w2d!_Nv&)T95xWV8@f;CmrYelt@gnCRP`Gx>EQK*0K26?^hcYmd5%}< zgO_r|WFFc2TE>opI>!S;nhgi-x5-@L?4y*8&)tC`t&dmoeQwv^s(Gfyhq+I0KG@=7 z>wJn!Xy2;#ZV$VPEp3juASpZ3a$b0UqR3U@J$7w=F$_rb-_G-qFaSX!sPKX$Hccuy zNu?Mk5W4$=G?>b90AJAXuQBIuECdj1XuKH=rkDoOvaP>6&B0*NYt~WvTn9BF*4Dx? z{n~7YxpbC#3=Fu<|HlZ?jM_I)`NZ6`C8(h{4)`!1`6}EKf_C`HCjy#ts1awmm`vdg zW|kMF@LYH7+swT&D$EWy=upCH%^fXhXUG}uSOAtzr7a9A@$aVjLSuD>h3?L}gP#O< zu--h!sH-xc_#y+EB*V^1HpvQuTZ4nlG2J5n+h7AsQQy<3<)GgxXn=ch9 zKvvjHrDK$^2;`%4RynweKFCe?--enRR|o=%l?){owcT6|ra@qV6%L}WK`v(~uyX=; z(R)tA%Hiw-9+mSt{o ztHb|cV22I?fvHz||MXF-4|nwzXxdgzA@SNF@!^U~z6_{NdO;RgcLeqE_S5SZu)q|v#`1`rS*gj%>;DC0j8)|T literal 0 HcmV?d00001 diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/common.py b/GPU-Virtual-Service/gpu-remoting/src/storage/common.py new file mode 100644 index 0000000..1220ea1 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/common.py @@ -0,0 +1,104 @@ +import json +import torch +import ctypes +import logging +import pickle +import struct + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +logging.getLogger("PIL.TiffImagePlugin").setLevel(51) + +EPOCH_STOP = 0 +DATALOADER_REQ = 1 +EPOCH_REQ = 2 + +class DataloaderInfo: + def __init__(self, train=True, remote_dataset_path=None, regular_dataset=None, batch_size=1, shuffle=False, sampler=None, batch_sampler=None, collate_fn=None, drop_last=False, preprocess=None, generator=None, num_workers=0, other_dataloader=None, distributed=False, seed=0, rank=0, world_size=1, sample_drop_last=False): + self.train = train + self.remote_dataset_path = remote_dataset_path + self.regular_dataset = regular_dataset + self.batch_size = batch_size + self.shuffle = shuffle + self.sampler = sampler + self.batch_sampler = batch_sampler + self.collate_fn = collate_fn + self.drop_last = drop_last + self.preprocess = preprocess + self.generator = generator + self.num_workers = num_workers + self.other_dataloader = other_dataloader + self.distributed = distributed + self.seed = seed + self.rank = rank + self.world_size = world_size + self.sample_drop_last = sample_drop_last + + def __str__(self): + return f"\nDataloaderInfo(train: {self.train}, remote_dataset_path: {self.remote_dataset_path}, batch_size: {self.batch_size}, shuffle: {self.shuffle}, sampler: {self.sampler}, batch_sampler: {self.batch_sampler}, collate_fn: {self.collate_fn}, drop_last: {self.drop_last}, preprocess: {self.preprocess}), generator: {self.generator}, num_workes: {self.num_workers},\ndistributed: {self.distributed}, seed: {self.seed}, rank: {self.rank}, world_size: {self.world_size}, sample_drop_last: {self.sample_drop_last},\nregular_dataset: {self.regular_dataset},\nother_dataloader: {self.other_dataloader}" + +class BatchInfo: + def __init__(self, train, batch_num, data_dtype=None, label_dtype=None, first_data_shape=None, first_label_shape=None, last_data_shape=None, last_label_shape=None): + self.train = train + self.batch_num = batch_num + self.data_dtype = data_dtype + self.label_dtype = label_dtype + self.first_data_shape = first_data_shape + self.first_label_shape = first_label_shape + self.last_data_shape = last_data_shape + self.last_label_shape = last_label_shape + + + def __str__(self): + return f"BatchInfo(train: {self.train}, batch_num: {self.batch_num}, data_dtype: {self.data_dtype}, label_dtype: {self.label_dtype}, first_data_shape: {self.first_data_shape}, first_label_shape: {self.first_label_shape}, last_data_shape: {self.last_data_shape}, last_label_shape: {self.last_label_shape}" + +class DatasetMetadata: + def __init__(self, train, batch_num, is_iterable, first_batch_metadata, last_batch_metadata): + self.train = train + self.batch_num = batch_num + self.is_iterable = is_iterable + self.first_batch = first_batch_metadata + self.last_batch = last_batch_metadata + + def __str__(self): + return f"DatasetMetadata(train: {self.train}, batch_num: {self.batch_num}, is_iterable:{self.is_iterable} ,first_batch: {self.first_batch}, last_batch: {self.last_batch}" + + +def get_server_ep(): + with open('config.json') as f: + config = json.load(f) + server_port = config['DatasetHandlerConfig']['handlerPort_'] + server_ip = config['DatasetHandlerConfig']['handlerIp_'] + return server_ip, server_port + +def get_client_id(): + with open('config.json') as f: + config = json.load(f) + return config['ClientConfig']['clientID_'] + +def get_size_t_len(): + return ctypes.sizeof(ctypes.c_size_t) + +def send_msg(socket, msg): + msg_byte = pickle.dumps(msg) + msg_len = len(msg_byte) + length_data = struct.pack('Q', msg_len) + socket.sendall(length_data) + socket.sendall(msg_byte) + + +def recv_msg(socket): + length_data = socket.recv(get_size_t_len()) + if not length_data: + return None + msg_len = struct.unpack('Q', length_data)[0] + msg_byte = bytearray(msg_len) + view = memoryview(msg_byte) + + bytes_received = 0 + while bytes_received < msg_len: + packet_size = socket.recv_into(view[bytes_received:]) + if not packet_size: + return None + bytes_received += packet_size + + return pickle.loads(msg_byte) \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/dataFeeder.py b/GPU-Virtual-Service/gpu-remoting/src/storage/dataFeeder.py new file mode 100644 index 0000000..5e654af --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/dataFeeder.py @@ -0,0 +1,258 @@ +import threading +import numpy as np +from PyShmqueue import * +import torch +import mmap +import posix_ipc +from torch.utils.data import DataLoader +# from torchvision import datasets +import torchvision +from common import DatasetMetadata, get_size_t_len, logging, DataloaderInfo +# from datasets import load_dataset, load_from_disk +import transformers +from transformers import default_data_collator + +class DataFeeder: + def __init__(self, ident="163_1234", identNum=1, + dataloaderInfo: DataloaderInfo = None): + torch.set_num_threads(8) + self.train = dataloaderInfo.train + self.ident = ident + self.identNum = identNum + self.stop_flag = threading.Event() + self.thread = None + + self.distributed = dataloaderInfo.distributed + + if dataloaderInfo.other_dataloader is not None: + self.dataloader = dataloaderInfo.other_dataloader + else: + # from datasets import load_from_disk + # if "ImageNet" in dataloaderInfo.remote_dataset_path: + # dataset_path = "/mnt/nvme0/FlexGV_Test/ImageNet-1K/{}".format("train" if self.train else "val") + # self.dataset = torchvision.datasets.ImageFolder(dataset_path, transform=dataloaderInfo.preprocess) + # elif "GLUE" in dataloaderInfo.remote_dataset_path: + # dataset_path = "/mnt/nvme0/FlexGV_Test/GLUE/MRPC_Processed" + # local_prep_dataset = load_from_disk(dataset_path) + # self.dataset = local_prep_dataset["train"] if self.train else local_prep_dataset["validation"] + if dataloaderInfo.regular_dataset is not None: + self.dataset = dataloaderInfo.regular_dataset + else: + self.dataset = torchvision.datasets.ImageFolder(dataloaderInfo.remote_dataset_path, transform=dataloaderInfo.preprocess) + + if self.distributed: + dataloaderInfo.sampler = torch.utils.data.distributed.DistributedSampler(dataset=self.dataset, + num_replicas=dataloaderInfo.world_size, + rank=dataloaderInfo.rank, shuffle=dataloaderInfo.shuffle, + seed=dataloaderInfo.seed, drop_last=dataloaderInfo.sample_drop_last) + + self.dataloader = DataLoader(self.dataset, batch_size=dataloaderInfo.batch_size, + shuffle=(dataloaderInfo.sampler is None), + sampler=dataloaderInfo.sampler, + batch_sampler=dataloaderInfo.batch_sampler, + collate_fn=dataloaderInfo.collate_fn, + drop_last=dataloaderInfo.drop_last, + generator=dataloaderInfo.generator, num_workers=dataloaderInfo.num_workers) + self.client_batch_cnt = len(self.dataloader) + self.dataloader_iter = iter(self.dataloader) + self.is_iterable = True + self.cur_batch_index = 0 + self.queue = None + self.shm_obj = None + self.shm_desc = None + self.queue_batch_num = 2 + + def reset(self, epoch = 0): + if self.distributed: + self.dataloader.sampler.set_epoch(epoch) + self.dataloader_iter = iter(self.dataloader) + self.cur_batch_index = 0 + self.client_batch_cnt = len(self.dataloader) + logging.debug(f"DataFeeder reset: isTrain: {self.train}, client_batch_cnt: {self.client_batch_cnt}") + if self.queue is not None: + self.queue.clear() + self.queue.ready() + + + def _create_shm_queue(self, first_batch_data): + size_t_len = get_size_t_len() + # queue_size = self.queue_batch_num * (data.numel()*data.element_size()+label.numel()*label.element_size() + 2*size_t_len) + first_batch_size = 0 + if self.is_iterable == False: + first_batch_data = first_batch_data.values() + for value in first_batch_data: + # first_batch_size += (value.numel()*value.element_size() + size_t_len) + first_batch_size += (value.numel()*value.element_size() + size_t_len * 2) + queue_size = self.queue_batch_num * first_batch_size + + # record the size of shared memory queue + data_type = 1 if self.train else 2 + # self.shm_obj = posix_ipc.SharedMemory(name=("/flexgv_shm_%d_%d" % (self.client_id, data_type)), + # flags=posix_ipc.O_CREAT, size=size_t_len) + self.shm_obj = posix_ipc.SharedMemory(name=("/flexgv_shm_%s_%d" % (self.ident, data_type)), # clientID_PID_1/2 + flags=posix_ipc.O_CREAT, size=size_t_len * 3) + self.shm_desc = mmap.mmap(self.shm_obj.fd, self.shm_obj.size) + self.shm_desc.seek(0) + logging.debug(f"first_batch_size={first_batch_size}, queue_size={queue_size}, client_batch_cnt={self.client_batch_cnt}, identNum={self.identNum}, ident={self.ident}") + self.shm_desc.write(queue_size.to_bytes(size_t_len, byteorder='little')) + self.shm_desc.write(self.client_batch_cnt.to_bytes(size_t_len, byteorder='little')) + self.shm_desc.write(self.identNum.to_bytes(size_t_len, byteorder='little')) + + # get the instance of shared memory queue + self.queue = ShareMemoryQueue((self.identNum<<2) + data_type, + queue_size, ONE_READ_ONE_WRITE, 1) # 1 for creating a new queue + logging.info(f"Shared Memory Queue with size: {queue_size} created for client({self.ident}), with data type = {data_type}") + + + def get_batch_info(self, is_first=False): + # first_batch_data, first_batch_label = next(iter(self.dataloader_iter)) + first_batch_data = next(iter(self.dataloader_iter)) + if isinstance(first_batch_data, dict): + self.is_iterable = False + + first_batch_metadata, last_batch_metadata = [], [] + for col in first_batch_data: + col_metadata = {} + if self.is_iterable: + col_metadata["key"] = None + else: + col_metadata["key"] = col + col = first_batch_data[col] + col_metadata["dtype"] = col.dtype + col_metadata["shape"] = col.shape + + col_metadata_for_last = col_metadata.copy() + if self.dataloader.drop_last == False: + if hasattr(self.dataloader.sampler, 'num_samples'): # DistributedSampler has num_samples + # number of samples that the current process should process + local_dataset_size = self.dataloader.sampler.num_samples + local_last_batch_size = local_dataset_size % self.dataloader.batch_size or self.dataloader.batch_size + col_metadata_for_last["shape"] = (local_last_batch_size, *col.shape[1:]) + else: + # single GPU + last_batch_size = len(self.dataloader.dataset) % self.dataloader.batch_size or self.dataloader.batch_size + col_metadata_for_last["shape"] = (last_batch_size, *col.shape[1:]) + + first_batch_metadata.append(col_metadata) + last_batch_metadata.append(col_metadata_for_last) + + # if self.dataloader.drop_last: + # last_data_shape, last_label_shape = first_batch_data.shape, first_batch_label.shape + # else: + # last_batch_size = len(self.dataloader.dataset) % self.dataloader.batch_size or self.dataloader.batch_size + # last_data_shape = (last_batch_size, *first_batch_data.shape[1:]) + # last_label_shape = (last_batch_size, *first_batch_label.shape[1:]) + + if is_first == True: + self._create_shm_queue(first_batch_data) + else: + if self.is_iterable == False: + first_batch_data = first_batch_data.values() + for value in first_batch_data: + self.queue.write(value.cpu().numpy()) + logging.info(f"[client: {self.ident}] preload_1st_data (isTrain: {self.train}): batch_idx = {self.cur_batch_index} / {self.queue_batch_num - 1}") + self.cur_batch_index += 1 # write the first batch + self.preload_data() # write the next batches + + # return BatchInfo(self.train, len(self.dataloader), first_batch_data.dtype, first_batch_label.dtype, first_batch_data.shape, first_batch_label.shape, last_data_shape, last_label_shape) + return DatasetMetadata(self.train, len(self.dataloader), self.is_iterable, first_batch_metadata, last_batch_metadata) + + def preload_data(self): + while self.cur_batch_index < self.queue_batch_num: + data = next(self.dataloader_iter) + if self.is_iterable == False: + data = data.values() + for value in data: + self.queue.write(value.cpu().numpy()) + logging.info(f"[client: {self.ident}] preload_data (isTrain: {self.train}): batch_idx = {self.cur_batch_index} / {self.queue_batch_num - 1}") + self.cur_batch_index += 1 # write the second batch + + def load_data(self): + # data_time_list = [] + while self.cur_batch_index < self.client_batch_cnt and not self.stop_flag.is_set(): + # start_time = time.time() + data = next(self.dataloader_iter) + # data_time = time.time() - start_time + if self.is_iterable == False: + data = data.values() + for value in data: + self.queue.write(value.cpu().numpy()) + # data_time_list.append(data_time) + # for value in data: + # self.queue.read(value.numel()*value.element_size()) + # logging.info(f"data time: {data_time}") + # logging.debug(f"[client: {self.ident}] load_data (isTrain: {self.train}): batch_idx = {self.cur_batch_index} / {self.client_batch_cnt - 1}") + + if (self.cur_batch_index % 10 == 0): + logging.info(f"[client: {self.ident}] load_data (isTrain: {self.train}) for 10 batch: cur_batch_idx = {self.cur_batch_index} / {self.client_batch_cnt - 1}") + self.cur_batch_index += 1 + logging.info(f"[client: {self.ident}] load_data: dataloader(isTrain: {self.train}) has been iterated over successfully, current batch index = {self.cur_batch_index - 1}") + # logging.info(f"[client: {self.ident}] load_data: data time Avg: {sum(data_time_list)/len(data_time_list)}") + # no need to reset the dataloader, since cudaMemcpyAsync may have not been finished + + def start_load_data(self): + if self.thread is None or not self.thread.is_alive(): + self.stop_flag.clear() + self.thread = threading.Thread(target=self.load_data) + self.thread.start() + logging.info(f"[{self.ident}] started loading data(isTrain={self.train})") + else: + logging.info(f"[{self.ident}] data loading thread(isTrain={self.train}) is already running") + + def stop_load_data(self, batch_num = 0): + if self.thread is not None and self.thread.is_alive(): + if batch_num > 0: + if self.cur_batch_index < batch_num: # wait for completing the batch_num + self.client_batch_cnt = batch_num + else: # already iterated over the batch_num + self.stop_flag.set() + self.queue.clear() # avoid blocking in the writing queue + else: + self.stop_flag.set() + self.thread.join() + self.thread = None + logging.info(f"[{self.ident}] stopped loading data(isTrain={self.train})") + else: + logging.info(f"[{self.ident}] data loading thread(isTrain={self.train}) has already been stopped") + + # def feed_data(self): + # data_type = 1 if self.train else 2 + # queue_size = None + # data, label = next(self.dataloader_iter) + # if self.queue is None: + # queue_size = 2 * (data.numel()*data.element_size()+label.numel()*label.element_size() + 2*get_size_t_len()) + # data_type = 1 if self.train else 2 + # size_t_len = get_size_t_len() + # self.shm_obj = posix_ipc.SharedMemory(name=("/flexgv_shm_%d_%d" % (self.client_id, data_type)), + # flags=posix_ipc.O_CREAT, size=size_t_len) + # self.shm_desc = mmap.mmap(self.shm_obj.fd, self.shm_obj.size) + # self.shm_desc.write(queue_size.to_bytes(size_t_len, byteorder='little')) + # self.queue = ShareMemoryQueue((self.client_id<<2) + data_type, + # queue_size, ONE_READ_ONE_WRITE, 1) # 0 for not creating a new queue + # print(f"Queue with size: {queue_size} created for client{self.client_id}, with data type = {data_type}") + + # # label_bytes = label.numpy().view(np.uint8) + # # print(f"is_train: {self.train} y_bytes[:8]:{label_bytes[:8].tobytes().hex()}, y_bytes[-8:]:{label_bytes[-8:].tobytes().hex()}") + # # self.queue.PrintTrunk() + # ret = self.queue.write(data.cpu().numpy()) + # # print(f"data write ret: {ret}") + # ret = self.queue.write(label.cpu().numpy()) + # # print(f"label write ret: {ret}") + # # ret = self.queue.write(label.cpu().numpy()) + # # print(f"test third wirte: {ret}") + + # res_idx = self.batch_index + # self.batch_index += 1 + # if (self.batch_index % 50 == 0): + # print(f"HYF: feed_data: batch_idx: {self.batch_index} / {len(self.dataloader) - 1}") + # # if self.batch_index >= len(self.dataloader): + # # self.reset() + + # # return BatchInfo(self.train, res_idx, data.shape, data.dtype, label.shape, label.dtype) + + def __del__(self): + self.is_closed = False + if self.shm_obj is not None: + self.shm_desc.close() + self.shm_obj.close_fd() + # self.shm_obj.unlink() # unlinked by the CUDA handle \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/dataloaderWrapper.py b/GPU-Virtual-Service/gpu-remoting/src/storage/dataloaderWrapper.py new file mode 100644 index 0000000..3287578 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/dataloaderWrapper.py @@ -0,0 +1,220 @@ +import logging +import sys +import socket +import mmap +import posix_ipc +import torch +import os +import struct +from torch.utils.data import DataLoader, Dataset +from torch import nn +from torchvision import datasets +from torchvision.transforms import ToTensor +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from common import DataloaderInfo, BatchInfo, get_server_ep, get_client_id, logging, send_msg, recv_msg, EPOCH_REQ, EPOCH_STOP, DATALOADER_REQ +import torch.distributed as dist + +from accelerate import Accelerator +import transformers +from transformers import ( + AutoConfig, + AutoModelForSequenceClassification, + AutoTokenizer, + DataCollatorWithPadding, + PretrainedConfig, + SchedulerType, + default_data_collator, + get_scheduler, +) + + +# logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +# REQUEST_TIMEOUT = 5000 +REQUEST_RETRIES = 3 + +pid = os.getpid() + +class ClientContext: + def __init__(self, server_ip, server_port, identity): + self.identity = identity + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.connect((server_ip, server_port)) + + def submit_dataloaderinfo(self, is_train, dataloader_info): + send_msg(self.socket, (DATALOADER_REQ, is_train, self.identity, dataloader_info)) + + def submit_epoch_req(self, is_train, epoch_idx): + send_msg(self.socket, (EPOCH_REQ, is_train, epoch_idx)) + + def stop_epoch(self, is_train, batch_idx): + send_msg(self.socket, (EPOCH_STOP, is_train, batch_idx)) + + def recv_dataset_metadata(self): + return recv_msg(self.socket) + + +client = None + +class FlexGVirtDataLoader(): + def __init__(self, train=True, # train or test + remote_dataset_path=None, regular_dataset=None, # remote dataset filepath or PyTorch built-in datasets + batch_size=1, # number of samples per batch + shuffle=False, # whether to shuffle the data at every epoch + sampler=None, # defines the strategy to draw samples from the dataset + batch_sampler=None, # defines the strategy to draw batches from the dataset (SequentialSampler, RandomSampler, WeightedSampler, etc.) + collate_fn=None, # merges a list of samples to form a mini-batch of Tensor(s) (e.g. image, label pairs) + drop_last=False, # whether to drop the last incomplete batch, if the dataset size is not divisible by the batch size + preprocess=None, # preprocessing function for each sample + generator=None, # random number generator + num_workers=0, # number of subprocesses to use for dataloading + other_dataloader=None, # other dataloader + proxy=True, # whether to use GPU proxy & Resource Dispatcher + distributed=False, # whether to use distributed data parallelism + seed=0, # random seed used to shuffle the sampler, identical across all processes in the distributed group + rank=None, # rank of the current process in the distributed group + world_size=None, # number of processes in the distributed group + sample_drop_last=False, # If ``False``, the sampler will add extra indices to make the data evenly divisible across the replicas. + ): + torch.cuda.mem_get_info() # todo + clientId = get_client_id() + identity = str(clientId) + "_" + str(pid) + logging.debug(f"dataloader identity: {identity}") + self.batch_num = 0 + self.train = train + + batch_info_bytes = struct.calcsize('bN') #! be careful with the alignment + self.shm_obj = posix_ipc.SharedMemory(name=("/flexgv_shm_%s_datatype" % identity), flags=posix_ipc.O_CREAT|posix_ipc.O_RDWR, size=(batch_info_bytes + 4 + 40)) # clientID_PID_datatype (contains: struct batch info + server ip and port) + self.shm_desc = mmap.mmap(self.shm_obj.fd, self.shm_obj.size) + self.shm_write_byte = (1 if self.train else 2).to_bytes(1, byteorder='little') # 1 for train, 2 for test/validation, 3 for model + self.shm_desc.seek(0) + self.shm_desc.write((0).to_bytes(1, byteorder='little')) # init the data type + + global client + if client is None: + if proxy: + self.shm_desc.seek(batch_info_bytes) + server_ip_port = self.shm_desc.read(4 + 40) # read server ip and port + server_port, server_ip_bytes = struct.unpack('i40s', server_ip_port) + server_ip = server_ip_bytes.decode('utf-8').strip('\x00') + else: # todo: to be validated + server_ip, server_port = get_server_ep(need_clientId = True) + client = ClientContext(server_ip, server_port, identity) + + self.remote_dataset_path = remote_dataset_path + self.dataset = regular_dataset + self.batch_size = batch_size + self.shuffle = shuffle + self.sampler = sampler + self.batch_sampler = batch_sampler + self.collate_fn = collate_fn + self.drop_last = drop_last + self.preprocess = preprocess + self.generator = generator + self.num_workers = num_workers + + self.distributed = distributed + self.seed = seed + self.epoch = 0 + if self.distributed: + if world_size is None: + if not dist.is_available(): + raise RuntimeError("Requires distributed package to be available") + world_size = dist.get_world_size() + if rank is None: + if not dist.is_available(): + raise RuntimeError("Requires distributed package to be available") + rank = dist.get_rank() + if rank >= world_size or rank < 0: + raise ValueError( + "Invalid rank {}, rank should be in the interval" + " [0, {}]".format(rank, world_size - 1)) + + self.rank = rank + self.world_size = world_size + logging.debug(f"rank={self.rank}, world_size={self.world_size}") + + logging.debug("MyDataLoader __init__ called") + client.submit_dataloaderinfo(self.train, DataloaderInfo(train, remote_dataset_path, regular_dataset, + batch_size, shuffle, sampler, batch_sampler, + collate_fn, drop_last, preprocess, + generator, num_workers, other_dataloader, + distributed, seed, rank, world_size, sample_drop_last)) + # self.batch_info = client.recv_batch_info() + self.dataset_metadata = client.recv_dataset_metadata() + self.batch_num = self.dataset_metadata.batch_num + self.stop_batch_idx = self.batch_num + self.set_epoch(0) + + logging.debug(f"Received dataset metadata(batch info): {self.dataset_metadata}") + + def set_epoch(self, epoch): + self.epoch = epoch + logging.debug(f"set_epoch called, epoch={self.epoch}") + + def stop(self): + self.stop_batch_idx = self.batch_idx # store the current batch index + self.batch_idx = self.batch_num # to stop the iteration + + def __iter__(self): + logging.debug(f"iter__init__ called, is_train={self.train}") + self.batch_idx = 0 + self.shm_desc.seek(0) + self.shm_desc.write(self.shm_write_byte) # notify the CUDA wrapper to start the data loading process + global client + client.submit_epoch_req(self.train, self.epoch) # notify the server to start the data loading process + # self.batch_info = client.recv_batch_info() # recv batch info again just for blocking the process + self.dataset_metadata = client.recv_dataset_metadata() # recv dataset metadata again just for blocking the process + self.batch_num = self.dataset_metadata.batch_num + self.stop_batch_idx = self.batch_num + + self.first_data = () if self.dataset_metadata.is_iterable else {} + self.first_batch_size = 0 + for col in self.dataset_metadata.first_batch: + data = torch.zeros(col["shape"], dtype=col["dtype"]) + self.first_batch_size += data.numel() * data.element_size() + if self.dataset_metadata.is_iterable: + self.first_data += (data,) + else: + self.first_data[col["key"]] = data + + self.last_data = () if self.dataset_metadata.is_iterable else {} + self.last_batch_size = 0 + for col in self.dataset_metadata.last_batch: + data = torch.zeros(col["shape"], dtype=col["dtype"]) + self.last_batch_size += data.numel() * data.element_size() + if self.dataset_metadata.is_iterable: + self.last_data += (data,) + else: + self.last_data[col["key"]] = data + + logging.debug(f"first_batch_size: {self.first_batch_size}, last_batch_size: {self.last_batch_size}") + + return self + + def __next__(self): + logging.debug(f"iter__next__ called, batch_idx={self.batch_idx}, batch_num={self.batch_num}") + if self.batch_idx < self.batch_num - 1: + self.batch_idx += 1 + self.shm_desc.seek(0) + self.shm_desc.write(struct.pack('bN', (1 if self.train else 2), self.first_batch_size)) + return self.first_data + elif self.batch_idx == self.batch_num - 1: + self.batch_idx += 1 + self.shm_desc.seek(0) + self.shm_desc.write(struct.pack('bN', (1 if self.train else 2), self.last_batch_size)) + return self.last_data + else: + self.shm_desc.seek(0) + self.shm_desc.write((0).to_bytes(1, byteorder='little')) # clear the data type + global client + client.stop_epoch(self.train, self.stop_batch_idx) # batch_idx indicates the next batch to be loaded + raise StopIteration + + def __len__(self): + return self.batch_num + + def __del__(self): + self.shm_desc.close() + self.shm_obj.close_fd() + + diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/preprocess_utils.py b/GPU-Virtual-Service/gpu-remoting/src/storage/preprocess_utils.py new file mode 100644 index 0000000..cc00643 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/preprocess_utils.py @@ -0,0 +1,134 @@ +import random +import numpy as np +import scipy.ndimage +from torch.utils.data import Dataset +from torchvision import transforms + + +def get_train_transforms(): + rand_flip = RandFlip() + cast = Cast(types=(np.float32, np.uint8)) + rand_scale = RandomBrightnessAugmentation(factor=0.3, prob=0.1) + rand_noise = GaussianNoise(mean=0.0, std=0.1, prob=0.1) + rand_crop = RandBalancedCrop(patch_size=[128, 128, 128], oversampling=0.4) + train_transforms = transforms.Compose([rand_crop, rand_flip, cast, rand_scale, rand_noise]) + return train_transforms + +class RandBalancedCrop: + def __init__(self, patch_size, oversampling): + self.patch_size = patch_size + self.oversampling = oversampling + + def __call__(self, data): + image, label = data["image"], data["label"] + if random.random() < self.oversampling: + image, label, cords = self.rand_foreg_cropd(image, label) + else: + image, label, cords = self._rand_crop(image, label) + data.update({"image": image, "label": label}) + return data + + @staticmethod + def randrange(max_range): + return 0 if max_range == 0 else random.randrange(max_range) + + def get_cords(self, cord, idx): + return cord[idx], cord[idx] + self.patch_size[idx] + + def _rand_crop(self, image, label): + ranges = [s - p for s, p in zip(image.shape[1:], self.patch_size)] + cord = [self.randrange(x) for x in ranges] + low_x, high_x = self.get_cords(cord, 0) + low_y, high_y = self.get_cords(cord, 1) + low_z, high_z = self.get_cords(cord, 2) + image = image[:, low_x:high_x, low_y:high_y, low_z:high_z] + label = label[:, low_x:high_x, low_y:high_y, low_z:high_z] + return image, label, [low_x, high_x, low_y, high_y, low_z, high_z] + + def rand_foreg_cropd(self, image, label): + def adjust(foreg_slice, patch_size, label, idx): + diff = patch_size[idx - 1] - (foreg_slice[idx].stop - foreg_slice[idx].start) + sign = -1 if diff < 0 else 1 + diff = abs(diff) + ladj = self.randrange(diff) + hadj = diff - ladj + low = max(0, foreg_slice[idx].start - sign * ladj) + high = min(label.shape[idx], foreg_slice[idx].stop + sign * hadj) + diff = patch_size[idx - 1] - (high - low) + if diff > 0 and low == 0: + high += diff + elif diff > 0: + low -= diff + return low, high + + cl = np.random.choice(np.unique(label[label > 0])) + foreg_slices = scipy.ndimage.find_objects(scipy.ndimage.measurements.label(label==cl)[0]) + foreg_slices = [x for x in foreg_slices if x is not None] + slice_volumes = [np.prod([s.stop - s.start for s in sl]) for sl in foreg_slices] + slice_idx = np.argsort(slice_volumes)[-2:] + foreg_slices = [foreg_slices[i] for i in slice_idx] + if not foreg_slices: + return self._rand_crop(image, label) + foreg_slice = foreg_slices[random.randrange(len(foreg_slices))] + low_x, high_x = adjust(foreg_slice, self.patch_size, label, 1) + low_y, high_y = adjust(foreg_slice, self.patch_size, label, 2) + low_z, high_z = adjust(foreg_slice, self.patch_size, label, 3) + image = image[:, low_x:high_x, low_y:high_y, low_z:high_z] + label = label[:, low_x:high_x, low_y:high_y, low_z:high_z] + return image, label, [low_x, high_x, low_y, high_y, low_z, high_z] + + +class RandFlip: + def __init__(self): + self.axis = [1, 2, 3] + self.prob = 1 / len(self.axis) + + def flip(self, data, axis): + data["image"] = np.flip(data["image"], axis=axis).copy() + data["label"] = np.flip(data["label"], axis=axis).copy() + return data + + def __call__(self, data): + for axis in self.axis: + if random.random() < self.prob: + data = self.flip(data, axis) + return data + + +class Cast: + def __init__(self, types): + self.types = types + + def __call__(self, data): + data["image"] = data["image"].astype(self.types[0]) + data["label"] = data["label"].astype(self.types[1]) + return data + + +class RandomBrightnessAugmentation: + def __init__(self, factor, prob): + self.prob = prob + self.factor = factor + + def __call__(self, data): + image = data["image"] + if random.random() < self.prob: + factor = np.random.uniform(low=1.0-self.factor, high=1.0+self.factor, size=1) + image = (image * (1 + factor)).astype(image.dtype) + data.update({"image": image}) + return data + + +class GaussianNoise: + def __init__(self, mean, std, prob): + self.mean = mean + self.std = std + self.prob = prob + + def __call__(self, data): + image = data["image"] + if random.random() < self.prob: + scale = np.random.uniform(low=0.0, high=self.std) + noise = np.random.normal(loc=self.mean, scale=scale, size=image.shape).astype(image.dtype) + data.update({"image": image + noise}) + return data \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/shareMemoryPy.bak b/GPU-Virtual-Service/gpu-remoting/src/storage/shareMemoryPy.bak new file mode 100644 index 0000000..c90ad79 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/shareMemoryPy.bak @@ -0,0 +1,89 @@ +import traceback +import numpy as np +import mmap +import os +import posix_ipc +import pdb + +shared_memory_size = 4096 * 4096 * 4096 +shared_memory_model = "shm_model" +shared_memory_data = "shm_data" +import mmap +import numpy as np +import posix_ipc +import multiprocessing + + +class SharedMemoryObj: + def __init__(self, name, size): + try: + self.shm = posix_ipc.SharedMemory(name=name, flags=posix_ipc.O_CREAT, size=size) + except posix_ipc.ExistentialError: + self.shm = posix_ipc.SharedMemory(name=name) + + # 将共享内存映射到当前进程的地址空间 + self.mapped_memory = mmap.mmap(self.shm.fd, self.shm.size) + # 创建锁 + self.lock = multiprocessing.Lock() + + def overwrite(self, batch): + if not isinstance(batch, np.ndarray): + raise ValueError("Input must be a NumPy array") + + with self.lock: + self.mapped_memory.seek(0) + bytes_batch = batch.tobytes() + self.mapped_memory.write(bytes_batch) + self.mapped_memory.flush() + print("Data overwritten to shared memory.") + + def append(self, batch): + if not isinstance(batch, np.ndarray): + raise ValueError("Input must be a NumPy array") + + with self.lock: + # 获取当前文件指针位置 + current_position = self.mapped_memory.tell() + # 将文件指针移到当前位置 + self.mapped_memory.seek(current_position) + bytes_batch = batch.tobytes() + self.mapped_memory.write(bytes_batch) + self.mapped_memory.flush() + print("Data appended to shared memory.") + + def read(self, size, offset): + with self.lock: + self.mapped_memory.seek(offset) + return self.mapped_memory.read(size) + + def close(self): + """ + Unlink and close the shared memory. + """ + try: + # Unlink the shared memory + self.shm.unlink() + # Close the mapped memory + self.mapped_memory.close() + except Exception as e: + print(f"Error: {e}") + + + +import numpy as np +import time + +def write_data_to_shared_memory(): + shared_memory = SharedMemoryObj(name='/example_shared_memory', size=1024) + + try: + for i in range(5): + data_to_write = np.array([i, i + 1, i + 2], dtype=np.int32) + shared_memory.append(data_to_write) + time.sleep(2) # Simulate other operations + + finally: + shared_memory.close() + +# if __name__ == "__main__": +# write_data_to_shared_memory() \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/src/storage/storageEndpoint.py b/GPU-Virtual-Service/gpu-remoting/src/storage/storageEndpoint.py new file mode 100644 index 0000000..75a0a65 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/src/storage/storageEndpoint.py @@ -0,0 +1,89 @@ +import socket +import errno +# import multiprocessing as mp +import torch.multiprocessing as mp +from dataFeeder import DataFeeder +from common import DataloaderInfo, BatchInfo, get_server_ep, logging, send_msg, recv_msg, EPOCH_REQ, EPOCH_STOP, DATALOADER_REQ + +def handle_client(client_socket, client_addr, rank): + logging.info(f"Accepted connection from {client_addr}") + datafeeders = [None, None] + client_ident = None + + try: + while True: + infoObj = recv_msg(client_socket) + if infoObj is None: + break + + request_type = infoObj[0] + is_train = infoObj[1] + if request_type == DATALOADER_REQ: + if client_ident is None: + client_ident = infoObj[2] + dataloader_info = infoObj[3] + logging.debug(f"[{client_ident}] new dataloader info: {dataloader_info}") + + datafeeder = DataFeeder(client_ident, rank, dataloader_info) + datafeeders[is_train] = datafeeder + # create the shared memory queue + send_msg(client_socket, datafeeder.get_batch_info(is_first=True)) + + elif request_type == EPOCH_REQ: + client_epoch_idx = infoObj[2] + logging.info(f"[{client_ident}] new epoch request (train={is_train}, epoch={client_epoch_idx})") + datafeeders[is_train].reset(client_epoch_idx) + send_msg(client_socket, datafeeders[is_train].get_batch_info(is_first=False)) + datafeeders[is_train].start_load_data() + elif request_type == EPOCH_STOP: + client_batch_idx = infoObj[2] + logging.info(f"[{client_ident}] epoch stop request (train={is_train}, client_batch_idx={client_batch_idx})") + datafeeders[is_train].stop_load_data(client_batch_idx) + else: + logging.error(f"Received unknown message: {infoObj}") + except socket.error as e: + if e.errno == errno.ECONNRESET: + pass + else: + logging.error(f"Unexpected socket error: {e}") + except Exception as e: + logging.error(f"Exception: {e}") + finally: + for datafeeder in datafeeders: + if datafeeder is not None: + datafeeder.stop_load_data() + logging.info(f"Connection from client#{client_ident}{client_addr} closed") + client_socket.close() + + + +def listener(server_address): + conn_num = 0 + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + server_socket.bind(server_address) + except OSError as e: + if e.errno == errno.EADDRINUSE: + logging.error(f"Address {server_address[0]}:{server_address[1]} is already in use") + return + else: + logging.error(f"Unexpected error: {e}") + return + server_socket.listen() + logging.info(f"Listening on {server_address[0]}:{server_address[1]}") + try: + while True: + client_socket, client_addr = server_socket.accept() + conn_num += 1 + process = mp.Process(target=handle_client, args=(client_socket, client_addr, conn_num)) + process.start() + client_socket.close() # close the socket in parent process, let the child process handle dataloading + except KeyboardInterrupt: + logging.info("Shutting down...") + except Exception as e: + logging.error(f"Error: {e}") + finally: + server_socket.close() + +if __name__ == "__main__": + listener(get_server_ep()) \ No newline at end of file