diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/MOILP_latency_simu.py b/GPU-Virtual-Service/gpu-remoting/scheduler/MOILP_latency_simu.py new file mode 100644 index 0000000..1c6dac4 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/MOILP_latency_simu.py @@ -0,0 +1,182 @@ +import random +import time +from tqdm import tqdm +import pulp + + +#测试单目标求解拓展性 +# 生成模拟数据：GPU 和任务 +def generate_gpu_tasks(gpu_count, task_per_gpu=2): + gpus = [f"gpu_{i}" for i in range(gpu_count)] + gpu_free_memory = {gpu: 10 for gpu in gpus} # 初始显存 10 + tasks = {} + task_id = 0 + + for gpu in gpus: + num_tasks = random.randint(1, 3) + for _ in range(num_tasks): + mem = random.randint(1, 5) # 任务占用显存 [1, 5] + gpu_free_memory[gpu] -= mem # 更新空闲显存 + tasks[task_id] = (gpu, mem, random.randint(1, 10)) # 服务量随机 + task_id += 1 + + # 确保空闲显存非负 + for gpu in gpus: + gpu_free_memory[gpu] = max(0, gpu_free_memory[gpu]) + + return gpus, tasks, gpu_free_memory + +# 贪心分配 GPU +def allocate_gpus(gpu_free_memory, m, k): + start_time = time.time() + available_gpus = {gpu: free for gpu, free in gpu_free_memory.items() if free >= m} + + if len(available_gpus) < k: + return None, time.time() - start_time + + allocated_gpus = list(available_gpus.keys())[:k] + total_time = time.time() - start_time + return allocated_gpus, total_time + +# 实验主函数 +def run_allocation_experiment(new_task_counts=[1, 5, 10]): + gpu_sizes = [100, 1000, 2500, 5000, 10000] + m = 5 # 每个 GPU 需提供 5 显存 + k_single = 2 # 每个新任务需要 2 个 GPU + results = [] + + print("开始空闲 GPU 分配扩展性实验...") + for gpu_count in tqdm(gpu_sizes): + gpus, tasks, gpu_free_memory = generate_gpu_tasks(gpu_count) + task_count = len(tasks) + + for N in new_task_counts: + k = N * k_single # 总 GPU 需求 + if k > gpu_count: + print(f"\nGPU={gpu_count}, N={N}: k={k} 超出 GPU 总数，跳过") + continue + + allocated_gpus, alloc_time = allocate_gpus(gpu_free_memory, m, k) + if allocated_gpus is None: + print(f"\nGPU={gpu_count}, N={N}: 无足够空闲 GPU") + continue + + results.append({ + "GPU Count": gpu_count, + "Task Count": task_count, + "New Tasks (N)": N, + "k": k, + "Allocation Time (s)": alloc_time + }) + print(f"\nGPU={gpu_count}, Tasks={task_count}, New Tasks={N}, k={k}:") + print(f" Allocation Time={alloc_time:.6f}s") + + # 输出结果表格 + print("\n实验结果汇总：") + print("| GPU Count | Task Count | New Tasks (N) | k | Allocation Time (s) |") + print("|-----------|------------|---------------|-----|---------------------|") + for res in results: + print(f"| {res['GPU Count']:<9} | {res['Task Count']:<10} | {res['New Tasks (N)']:<13} | {res['k']:<3} | {res['Allocation Time (s)']:<19.6f} |") + +# 运行实验 +# if __name__ == "__main__": +# run_allocation_experiment(new_task_counts=[1, 5, 10]) + + + +#测试多目标求解 + + +# import random +# import time +# from tqdm import tqdm # 用于显示进度条 + +# 生成模拟数据的函数 +def generate_tasks(gpu_count, task_per_gpu=2): + gpus = [f"gpu_{i}" for i in range(gpu_count)] + tasks = {} + task_id = 0 + for gpu in gpus: + # 每个 GPU 随机 1-3 个任务，平均约 2 个 + num_tasks = random.randint(1, 3) + for _ in range(num_tasks): + mem = random.randint(1, 10) # 显存 [1, 10] + svc = random.randint(1, 10) # 服务量 [1, 10] + tasks[task_id] = (gpu, mem, svc) + task_id += 1 + return gpus, tasks + +# MOILP 求解函数 +def solve_moilp(gpus, tasks, m, k): + # 第一阶段：最小化任务数量 + start_time = time.time() + prob1 = pulp.LpProblem("Minimize_Task_Count", pulp.LpMinimize) + x = {t: pulp.LpVariable(f"x_{t}", cat="Binary") for t in tasks} + y = {i: pulp.LpVariable(f"y_{i}", cat="Binary") for i in gpus} + prob1 += pulp.lpSum(x[t] for t in tasks) + for i in gpus: + tasks_on_gpu_i = [t for t in tasks if tasks[t][0] == i] + prob1 += pulp.lpSum(tasks[t][1] * x[t] for t in tasks_on_gpu_i) >= m * y[i] + prob1 += pulp.lpSum(y[i] for i in gpus) == k + prob1.solve(pulp.PULP_CBC_CMD(msg=0)) # msg=0 关闭求解日志 + stage1_time = time.time() - start_time + if pulp.LpStatus[prob1.status] != "Optimal": + return None, stage1_time, 0 + n_min = int(pulp.value(prob1.objective)) + + # 第二阶段：最大化服务量 + start_time = time.time() + prob2 = pulp.LpProblem("Maximize_Service_Quantity", pulp.LpMaximize) + x = {t: pulp.LpVariable(f"x_{t}", cat="Binary") for t in tasks} + y = {i: pulp.LpVariable(f"y_{i}", cat="Binary") for i in gpus} + prob2 += pulp.lpSum(tasks[t][2] * x[t] for t in tasks) + for i in gpus: + tasks_on_gpu_i = [t for t in tasks if tasks[t][0] == i] + prob2 += pulp.lpSum(tasks[t][1] * x[t] for t in tasks_on_gpu_i) >= m * y[i] + prob2 += pulp.lpSum(y[i] for i in gpus) == k + prob2 += pulp.lpSum(x[t] for t in tasks) == n_min + prob2.solve(pulp.PULP_CBC_CMD(msg=0)) + stage2_time = time.time() - start_time + + total_time = stage1_time + stage2_time + return n_min, stage1_time, stage2_time + +# 实验主函数 +def run_experiment(): + gpu_sizes = [10, 50, 100, 1000, 2500, 5000, 10000] + m = 5 + results = [] + + print("开始扩展性实验...") + for gpu_count in tqdm(gpu_sizes): + # k = int(0.1 * gpu_count) # 抢占 10% 的 GPU + k = 2 + gpus, tasks = generate_tasks(gpu_count) + task_count = len(tasks) + + n_min, stage1_time, stage2_time = solve_moilp(gpus, tasks, m, k) + total_time = stage1_time + stage2_time + + results.append({ + "GPU Count": gpu_count, + "Task Count": task_count, + "k": k, + "Min Tasks": n_min, + "Stage 1 Time (s)": stage1_time, + "Stage 2 Time (s)": stage2_time, + "Total Time (s)": total_time + }) + print(f"\nGPU={gpu_count}, Tasks={task_count}, k={k}:") + print(f" Min Tasks={n_min}, Total Time={total_time:.2f}s (Stage 1: {stage1_time:.2f}s, Stage 2: {stage2_time:.2f}s)") + + # 输出结果表格 + print("\n实验结果汇总：") + print("| GPU Count | Task Count | k | Min Tasks | Stage 1 Time (s) | Stage 2 Time (s) | Total Time (s) |") + print("|-----------|------------|-----|-----------|------------------|------------------|----------------|") + for res in results: + print(f"| {res['GPU Count']:<9} | {res['Task Count']:<10} | {res['k']:<3} | {res['Min Tasks'] or 'N/A':<9} | {res['Stage 1 Time (s)']:<16.2f} | {res['Stage 2 Time (s)']:<16.2f} | {res['Total Time (s)']:<14.2f} |") + +# 运行实验 +if __name__ == "__main__": + run_allocation_experiment(new_task_counts=[1, 5, 10]) + run_experiment() \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/__init__.py b/GPU-Virtual-Service/gpu-remoting/scheduler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/client_simulator.py b/GPU-Virtual-Service/gpu-remoting/scheduler/client_simulator.py new file mode 100644 index 0000000..551ee04 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/client_simulator.py @@ -0,0 +1,164 @@ +import os +import sys +import torch +import torch.nn as nn +from torchvision import models, transforms +from collections import deque +import time +import threading +import logging +# from typing import Dict, List, Any +from typing import Dict, List, Any, Optional + +# 将 scheduler 目录添加到 sys.path 中 +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.append(parent_dir) +import socket +import random + +import uuid + +# from scheduler.requese_handler_5 import * +from scheduler.util import * + + +# 设置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +workload1 = [ #低推理、低训练负载 + f"User_Reqeust_Training:user1,resnet18,{32},{1}", + f"User_Reqeust_Training:user2,resnet18,{32},{1}", + # f"User_Reqeust_Training:user3,mobilenet_v2,{32},{1}", + # f"wait", + + f"User_Request_Inference:user4,resnet18,{50},True,{100},{4}", + f"User_Request_Inference:user5,vgg16,{40},True,{100},{2}", + f"User_Request_Inference:user6,densenet121,{100},True,{100},{8}", + +] + +workload2 = [ #低推理、高训练负载 + + f"User_Reqeust_Training:user5,resnet50,{64},{1}", + f"User_Reqeust_Training:user6,resnet50,{32},{1}", + f"User_Reqeust_Training:user7,resnet50,{64},{1}", + f"User_Reqeust_Training:user8,resnet50,{32},{1}", + f"wait", + f"User_Request_Inference:user1,resnet18,{50},True,{100},{4}", + f"User_Request_Inference:user2,vgg16,{40},True,{100},{2}", + f"User_Request_Inference:user3,densenet121,{100},True,{100},{8}", + f"User_Request_Inference:user4,mobilenet_v2,{50},True,{100},{4}", + + # f"User_Reqeust_Training:user9,resnet50,{128},{1}" + # f"User_Reqeust_Training:user10,resnet50,{128},{1}" +] + +workload3 = [ #高推理、低训练负载 + # f"User_Reqeust_Training:user01,resnet18,{32},{1}", + # f"User_Reqeust_Training:user02,resnet18,{32},{1}", + # f"User_Reqeust_Training:user03,mobilenet_v2,{32},{1}", + # f"wait", + f"User_Request_Inference:user1,resnet18,{200},True,{100},{4}", + f"User_Request_Inference:user2,resnet18,{100},True,{100},{8}", + f"User_Request_Inference:user3,resnet18,{100},True,{100},{4}", + f"User_Request_Inference:user4,resnet18,{100},True,{100},{16}", + f"User_Request_Inference:user5,resnet18,{100},True,{100},{16}", + f"User_Request_Inference:user6,resnet18,{100},True,{100},{8}", + f"User_Request_Inference:user7,vgg16,{100},True,{100},{4}", + f"User_Request_Inference:user8,vgg16,{100},True,{100},{8}", + f"User_Request_Inference:user9,vgg16,{100},True,{100},{8}", + f"User_Request_Inference:user10,vgg16,{100},True,{100},{16}", + f"User_Request_Inference:user11,vgg16,{100},True,{100},{16}", + f"User_Request_Inference:user12,vgg16,{100},True,{100},{8}", + f"User_Request_Inference:user13,densenet121,{100},True,{100},{4}", + f"User_Request_Inference:user14,densenet121,{100},True,{100},{8}", + f"User_Request_Inference:user15,densenet121,{100},True,{100},{8}", + f"User_Request_Inference:user16,densenet121,{100},True,{100},{16}", + f"User_Request_Inference:user17,densenet121,{100},True,{100},{16}", + f"User_Request_Inference:user18,densenet121,{100},True,{100},{8}", +] + +workload4 = [ #高推理、高训练负载 + f"User_Reqeust_Training:user05,resnet50,{64},{1}", + f"User_Reqeust_Training:user06,resnet50,{64},{1}", + f"User_Reqeust_Training:user07,resnet18,{64},{1}", + f"User_Reqeust_Training:user08,resnet18,{64},{1}", + f"User_Reqeust_Training:user09,resnet50,{32},{1}", + f"User_Reqeust_Training:user010,resnet50,{32},{1}", + f"wait", + f"User_Request_Inference:user1,resnet18,{200},True,{100},{4}", + f"User_Request_Inference:user2,resnet18,{100},True,{100},{8}", + f"User_Request_Inference:user3,resnet18,{100},True,{100},{4}", + f"User_Request_Inference:user4,resnet18,{100},True,{100},{16}", + f"User_Request_Inference:user5,resnet18,{100},True,{100},{16}", + f"User_Request_Inference:user6,resnet18,{100},True,{100},{8}", + f"User_Request_Inference:user7,vgg16,{100},True,{100},{4}", + f"User_Request_Inference:user8,vgg16,{100},True,{100},{8}", + f"User_Request_Inference:user9,vgg16,{100},True,{100},{8}", + f"User_Request_Inference:user10,vgg16,{100},True,{100},{16}", + f"User_Request_Inference:user11,vgg16,{100},True,{100},{16}", + f"User_Request_Inference:user12,vgg16,{100},True,{100},{8}", + f"User_Request_Inference:user13,densenet121,{100},True,{100},{4}", + f"User_Request_Inference:user14,densenet121,{100},True,{100},{8}", + f"User_Request_Inference:user15,densenet121,{100},True,{100},{8}", + f"User_Request_Inference:user16,densenet121,{100},True,{100},{16}", + f"User_Request_Inference:user17,densenet121,{100},True,{100},{16}", + f"User_Request_Inference:user18,densenet121,{100},True,{100},{8}", + + +] + + +workload5 = [ #高推理、低训练负载，推理使用泊松分布 + # f"User_Reqeust_Training:user04,resnet18,{8},{1}", + # f"User_Reqeust_Training:user05,resnet18,{8},{1}", + # f"User_Reqeust_Training:user06,mobilenet_v2,{32},{1}" + f"wait", + f"User_Request_Inference:user1,resnet18,{200},False,{100},{4}", + f"User_Request_Inference:user2,resnet18,{100},False,{100},{8}", + f"User_Request_Inference:user3,resnet18,{100},False,{100},{4}", + f"User_Request_Inference:user4,resnet18,{100},False,{100},{16}", + f"User_Request_Inference:user5,resnet18,{100},False,{100},{16}", + f"User_Request_Inference:user6,resnet18,{100},False,{100},{8}", + f"User_Request_Inference:user7,vgg16,{100},False,{100},{4}", + f"User_Request_Inference:user8,vgg16,{100},False,{100},{8}", + f"User_Request_Inference:user9,vgg16,{100},False,{100},{8}", + f"User_Request_Inference:user10,vgg16,{100},False,{100},{16}", + f"User_Request_Inference:user11,vgg16,{100},False,{100},{16}", + f"User_Request_Inference:user12,vgg16,{100},False,{100},{8}", + f"User_Request_Inference:user13,densenet121,{100},False,{100},{4}", + f"User_Request_Inference:user14,densenet121,{100},False,{100},{8}", + f"User_Request_Inference:user15,densenet121,{100},False,{100},{8}", + f"User_Request_Inference:user16,densenet121,{100},False,{100},{16}", + f"User_Request_Inference:user17,densenet121,{100},False,{100},{16}", + f"User_Request_Inference:user18,densenet121,{100},False,{100},{8}", + + ] + + +def main(): + config = get_config_file() + glb_Ip = config["GlobalConfig"]["glbIp_"] + glb_Port = config["GlobalConfig"]["glbPort_"] + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((glb_Ip, glb_Port)) + logger.info(f"Connected to Global Server at {glb_Ip}:{glb_Port}") + # 发送请求 + + for msg in workload2: + if msg == "wait": + time.sleep(20) + continue + logger.info(f"Sending message: {msg}") + s.sendall(msg.encode()) + time.sleep(0.05) + + + while True: + time.sleep(1) + +if __name__ == "__main__": + main() + \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/cluster.py b/GPU-Virtual-Service/gpu-remoting/scheduler/cluster.py new file mode 100644 index 0000000..3bfbde4 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/cluster.py @@ -0,0 +1,55 @@ +import argparse +import socket +import threading +import json +import redis +from ctypes import Structure, c_char, c_int, c_size_t, sizeof +from cffi import FFI +import struct +import logging +from collections import deque +import time +from multiprocessing import Process +import os +import sys + +# 将 scheduler 目录添加到 sys.path 中 +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.append(parent_dir) + +#自定义类 +from scheduler.util import * +from scheduler.gpu_info import* +from scheduler.job import * +from scheduler.node import * + + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + +class Cluster: + def __init__(self): + self.node_list = [] + self.node_list_lock = threading.Lock() + self.all_job_list = [] + self.job_list_lock = threading.Lock() + self.preemt_job_list = [] + self.preemt_job_list_lock = threading.Lock() + self.running_job_list = [] + self.running_job_list_lock = threading.Lock() + self.waiting_job_list = [] + self.waiting_job_list_lock = threading.Lock() + + + def sort_node_list(self, type, free_gpu_type, rev): + if type == 1:#按照资源最空闲的节点排序 + if rev: + self.node_list.sort(key=lambda x: x.free_gpu_num(free_gpu_type), reverse=True) + else: + self.node_list.sort(key=lambda x: x.free_gpu_num(free_gpu_type)) + elif type == 2:#按照job的数量排序 + if rev: + self.node_list.sort(key=lambda x: x.total_job_num, reverse = True) + else: + self.node_list.sort(key=lambda x: x.total_job_num) \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/cluster_client_sender.py b/GPU-Virtual-Service/gpu-remoting/scheduler/cluster_client_sender.py new file mode 100644 index 0000000..37fc707 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/cluster_client_sender.py @@ -0,0 +1,206 @@ +import argparse +import socket +import threading +import json +import redis +from ctypes import Structure, c_char, c_int, c_size_t, sizeof +from cffi import FFI +import struct +import logging +from collections import deque +import time +from multiprocessing import Process +import subprocess +from datetime import datetime +import os + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +def run_command(command_str, log_file_path): + with open(log_file_path, 'w') as log_file: + subprocess.run(command_str, shell=True, stdout=log_file, stderr=log_file) + + +def handle_request(client_id, req_gpu_num, priority, model, type, batchsize, job_type, batch_rate, epoch, url = 0): + cv_command = [ + "FLEXGV_CLIENT_ID={}".format(client_id), + "FLEXGV_PRIORITY={}".format(priority), + "FLEXGV_REQ_NUM={}".format(req_gpu_num), + "FLEXGV_MODEL={}".format(model), + "FLEXGV_BATCH_SIZE={}".format(batchsize), + "LD_PRELOAD=./out/lib64/libcuda_hook.so", + "LD_LIBRARY_PATH=./out/lib64:$LD_LIBRARY_PATH", + "python", "scripts/workloads/imageNetTrain.py", + "-a", model, + "-b", str(batchsize), + "--gpu", "0", + "-j", "2", + "--epochs", str(epoch) + ] + if type == 1: + cv_command.append("--evaluate") + cv_command.append("--pretrained") + elif type == 0: + if batch_rate > 0: + cv_command.append("--batch_rate") + cv_command.append(f"{batch_rate}") + + gpt_command = [ + "FLEXGV_CLIENT_ID={}".format(client_id), + "FLEXGV_PRIORITY={}".format(priority), + "FLEXGV_REQ_NUM={}".format(req_gpu_num), + "FLEXGV_MODEL=gpt", + "FLEXGV_BATCH_SIZE={}".format(batchsize), + "LD_PRELOAD=./out/lib64/libcuda_hook.so", + "LD_LIBRARY_PATH=./out/lib64:$LD_LIBRARY_PATH", + "python", "scripts/workloads/large-language-model/clmTrainWoPrep.py", + "-b", str(batchsize), + "-e", str(epoch) + ] + + bert_command = [ + "FLEXGV_CLIENT_ID={}".format(client_id), + "FLEXGV_PRIORITY={}".format(priority), + "FLEXGV_REQ_NUM={}".format(req_gpu_num), + "FLEXGV_MODEL=BERT", + "FLEXGV_BATCH_SIZE={}".format(batchsize), + "LD_PRELOAD=./out/lib64/libcuda_hook.so", + "LD_LIBRARY_PATH=./out/lib64:$LD_LIBRARY_PATH", + "python", "scripts/workloads/text-classification/glueTrainWoPrep.py", + "-b", str(batchsize), + "-e", str(epoch), + ] + + ddp_command = [ + "FLEXGV_CLIENT_ID={}".format(client_id), + "FLEXGV_PRIORITY={}".format(priority), + "FLEXGV_REQ_NUM={}".format(req_gpu_num), + "FLEXGV_MODEL={}".format(model), + "FLEXGV_BATCH_SIZE={}".format(batchsize), + "LD_PRELOAD=./out/lib64/libcuda_hook.so", + "LD_LIBRARY_PATH=./out/lib64:$LD_LIBRARY_PATH", + "python", "scripts/workloads/imageNetTrainDDP.py", + "/home/djh/dataset/ImageNet-1K", + "-a", model, + "-b", str(batchsize), + "--epochs", str(epoch), + "--dist-url", f"tcp://127.0.0.1:166{url}" + ] + if type == 1: + ddp_command.append("--evaluate") + ddp_command.append("--pretrained") + elif type == 0: + if batch_rate > 0: + ddp_command.append("--batch_rate") + ddp_command.append(f"{batch_rate}") + + if job_type == 0: + command_str = " ".join(cv_command) + logging.info(f'Executing command: {command_str}') + elif job_type == 1: + command_str = " ".join(gpt_command) + logging.info(f'Executing command: {command_str}') + elif job_type == 2: + command_str = " ".join(bert_command) + logging.info(f'Executing command: {command_str}') + elif job_type == 3: + command_str = " ".join(ddp_command) + logging.info(f'Executing command: {command_str}') + + # 获取今天的日期 + date = datetime.now().strftime('%Y-%m-%d') + # 创建目录路径 + result_dir = f'result_{date}' + # 如果目录不存在，则创建目录 + if not os.path.exists(result_dir): + os.makedirs(result_dir) + # 设置日志文件路径 + log_file_path = os.path.join(result_dir, f'output{client_id}.log') + # with open(log_file_path, 'w') as log_file: + # # 执行命令并将输出重定向到日志文件 + # subprocess.run(command_str, shell=True, stdout=log_file, stderr=log_file) + + process = Process(target=run_command, args=(command_str, log_file_path)) + process.start() + return process + + +def main(): + # Trace 30 + handle_request(1001, 1, 0, "mobilenet_v2", 0, 32, 0, 0.05, 1) # 小 + time.sleep(1) + handle_request(1002, 1, 0, "GPT3", 0, 2, 1, 0.8, 4) # LLM, 大 + time.sleep(1) + handle_request(1003, 1, 0, "resnet50", 0, 32, 0, 0.3, 4) # 中 + time.sleep(1) + handle_request(1004, 1, 0, "Bert", 0, 64, 2, 0.3, 4) # NLP, 中 + time.sleep(200) + handle_request(1005, 1, 0, "shufflenet_v2_x0_5", 0, 32, 0, 0.05, 1) # CV, 小 + time.sleep(1) + handle_request(1006, 1, 0, "resnet18", 0, 64, 0, 0.3, 4) # 中 + time.sleep(1) + handle_request(1007, 1, 0, "GPT3", 0, 2, 1, 0.8, 2) # LLM, 大 + time.sleep(1) + handle_request(1008, 1, 0, "mobilenet_v2", 0, 32, 0, 0.05, 1) # CV, 小 + time.sleep(200) + handle_request(1009, 1, 0, "Bert", 0, 64, 2, 0.3, 4) # NLP, 中 + time.sleep(1) + handle_request(1010, 1, 0, "vgg16", 0, 64, 0, 0.3, 4) # 中 + time.sleep(1) + handle_request(1011, 1, 0, "shufflenet_v2_x0_5", 0, 32, 0, 0.05, 1) # CV, 小 + time.sleep(1) + handle_request(1012, 1, 0, "resnet18", 0, 32, 0, 0.05, 1) # CV, 小 + time.sleep(200) + handle_request(1013, 1, 0, "GPT3", 0, 2, 1, 0.8, 12) # LLM, 大 + time.sleep(1) + handle_request(1014, 1, 0, "mobilenet_v2", 0, 32, 0, 0.05, 1) # 小 + time.sleep(1) + handle_request(1015, 1, 0, "Bert", 0, 64, 2, 0.3, 4) # NLP, 中 + time.sleep(1) + handle_request(1016, 1, 0, "resnet50", 0, 64, 0, 0.3, 4) # CV, 中 + time.sleep(200) + + handle_request(1017, 1, 0, "shufflenet_v2_x0_5", 0, 32, 0, 0.05, 1) # CV, 小 + time.sleep(1) + handle_request(1018, 1, 0, "GPT3", 0, 2, 1, 0.05, 1) # LLM, 小 + time.sleep(1) + handle_request(1019, 2, 0, "vgg16", 0, 32, 0, 0.05, 1) # 小 + time.sleep(1) + handle_request(1020, 1, 0, "mobilenet_v2", 0, 32, 0, 0.05, 1) # CV, 小 + time.sleep(200) + handle_request(1021, 1, 0, "Bert", 0, 64, 2, 0.3, 4) # NLP, 中 + time.sleep(1) + handle_request(1022, 1, 0, "resnet50", 0, 32, 0, 0.05, 1) # CV, 小 + time.sleep(1) + handle_request(1023, 1, 0, "GPT3", 0, 2, 1, 0.05, 1) # LLM, 小 + time.sleep(1) + handle_request(1024, 1, 0, "shufflenet_v2_x0_5", 0, 32, 0, 0.05, 1) # CV, 小 + time.sleep(200) + handle_request(1025, 1, 0, "resnet18", 0, 32, 0, 0.05, 1) # CV, 小 + time.sleep(1) + handle_request(1026, 1, 0, "mobilenet_v2", 0, 32, 0, 0.05, 1) # CV, 小 + time.sleep(1) + handle_request(1027, 1, 0, "Bert", 0, 32, 2, 0.05, 1) # NLP, 小 + time.sleep(1) + handle_request(1028, 1, 0, "resnet50", 0, 64, 0, 0.3, 4) # CV, 中 + time.sleep(200) + handle_request(1029, 1, 0, "GPT3", 0, 2, 1, 0.05, 1) # LLM, 小 + time.sleep(1) + handle_request(1030, 1, 0, "vgg16", 0, 32, 0, 0.05, 1) # CV, 小 + # 按时间顺序启动任务 + processes = [] + ddp_counter = 0 # 用于生成递增的url + + # 可选：等待所有进程完成 + for p in processes: + p.join() + logging.info("All processes completed.") + +if __name__ == "__main__": + # 配置日志 + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + main() + + + + \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/global_scheduler.py b/GPU-Virtual-Service/gpu-remoting/scheduler/global_scheduler.py new file mode 100644 index 0000000..c92512d --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/global_scheduler.py @@ -0,0 +1,339 @@ +import os +import sys +import torch +import torch.nn as nn +from torchvision import models, transforms +from collections import deque +import time +import threading +import logging +# from typing import Dict, List, Any +from typing import Dict, List, Any, Optional + +# 将 scheduler 目录添加到 sys.path 中 +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.append(parent_dir) +import socket +import random + +import uuid + +# from scheduler.requese_handler_5 import * +from scheduler.util import * +from scheduler.gpu_info import* +from scheduler.job import * +from scheduler.node import * +#Version ： + + +# 设置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +class Domain: + def __init__(self, domain_id, ip, port, enable_batching, conn): + self.domain_id = domain_id + self.ip = ip + self.port = port + self.enable_batching = enable_batching + self.conn = conn + self.user_num = 0 + self.job_num = 0 + self.model_user_num: Dict[str, int]= {} + + def get_load(self, arch): + return self.model_user_num[arch] if arch in self.model_user_num else 0 + + +# Global Scheduler +class GlobalScheduler: + def __init__(self, domains: list, slo_latency: float = 0.5): + self.domains = {domain.domain_id: domain for domain in domains} + self.slo_latency = slo_latency + self.model_domain_map: Dict[str, List[str]] = {} + self.stop_event = threading.Event() + self.enable_batching = True + + + def get_idle_domain(self) -> Optional[str]: + if not self.domains: + return None # 如果没有可用 Domain，返回 None + + # 找到 user_num + job_num 最小的 Domain + min_load = float('inf') + idle_domain_id = None + + for domain_id, domain in self.domains.items(): + total_load = domain.user_num + domain.job_num + if total_load < min_load: + min_load = total_load + idle_domain_id = domain_id + + return idle_domain_id + + def start_user_requests(self, data): + message = data.split(":", 1)[1].split(',') + user_id = message[0] + arch = message[1] + rps = float(message[2]) + uniform = message[3].lower() == 'true' + num_requests = int(message[4]) + batch_size = int(message[5]) + msg = f"usereq:{user_id},{arch},{rps},{uniform},{num_requests},{batch_size}" + if self.enable_batching: + if arch in self.model_domain_map: + logger.info(f"Batching Starting user {user_id} requests for Arch {arch} in existing domain") + best_domain_id = None + min_latency = float('inf') + for domain_id in self.model_domain_map[arch]: + domain = self.domains[domain_id] + # current_latency = gateway.get_load(arch) + load_num = domain.get_load(arch) + if load_num <= 3: + best_domain_id = domain_id + logger.info(f"Best domain found: {best_domain_id} with load {load_num}") + break + # if min_latency <= self.slo_latency: + if best_domain_id is not None and self.domains[best_domain_id].user_num < 3: + logger.info(f"Starting user {user_id} requests in domain {best_domain_id} for Arch {arch} with P90 latency {min_latency:.4f}") + domain_conn = self.domains[best_domain_id].conn + self.domains[best_domain_id].user_num += 1 + self.domains[best_domain_id].model_user_num[arch] = self.domains[best_domain_id].get_load(arch) + 1 + domain_conn.sendall(msg.encode()) + else: + new_domain_id = self.get_idle_domain() + if new_domain_id: + if arch not in self.model_domain_map: + self.model_domain_map[arch] = [] + self.model_domain_map[arch].append(new_domain_id) + logger.info(f"Scaling Arch {arch} to new domain {new_domain_id} for user {user_id}") + domain_conn = self.domains[new_domain_id].conn + self.domains[new_domain_id].job_num += 1 + self.domains[new_domain_id].model_user_num[arch] = self.domains[new_domain_id].get_load(arch) + 1 + domain_conn.sendall(msg.encode()) + + else: + logger.warning(f"No idle domain available, using domain {best_domain_id} with P90 latency {min_latency:.4f}") + domain_conn = self.domains[best_domain_id].conn + self.domains[best_domain_id].model_user_num[arch] = self.domains[best_domain_id].get_load(arch) + 1 + # self.domains[best_domain_id].user_num += 1 + domain_conn.sendall(msg.encode()) + else: + domain_id = self.get_idle_domain() + if domain_id: + self.model_domain_map[arch] = [domain_id] + logger.info(f"Starting Arch {arch} in new domain {domain_id} for user {user_id}") + domain_conn = self.domains[domain_id].conn + self.domains[domain_id].job_num += 1 + self.domains[domain_id].model_user_num[arch] = self.domains[domain_id].get_load(arch) + 1 + # self.domains[domain_id].user_num += 1 + domain_conn.sendall(msg.encode()) + else: + domain_id = random.choice(list(self.domains.keys())) + self.model_domain_map[arch] = [domain_id] + logger.warning(f"No idle domain, assigning Arch {arch} to Domain {domain_id} for user {user_id}") + domain_conn = self.domains[domain_id].conn + self.domains[domain_id].job_num += 1 + self.domains[domain_id].model_user_num[arch] = self.domains[domain_id].get_load(arch) + 1 + # self.domains[domain_id].user_num += 1 + domain_conn.sendall(msg.encode()) + else: + domain_id = random.choice(list(self.domains.keys())) + logger.info(f"Non-batching mode: Starting user {user_id} requests in domain {domain_id} for Arch {arch}") + domain_conn = self.domains[domain_id].conn + self.domains[domain_id].job_num += 1 + self.domains[domain_id].user_num += 1 + domain_conn.sendall(msg.encode()) + + def start_Training_requests(self, data): + message = data.split(":", 1)[1].split(',') + user_id = message[0] + arch = message[1] + batch_size = int(message[2]) + epochs = int(message[3]) + domain = self.get_idle_domain() + if domain: + logger.info(f"Starting user {user_id} training requests in domain {domain} for Arch {arch}") + domain_conn = self.domains[domain].conn + self.domains[domain].job_num += 1 + self.domains[domain].user_num += 1 + msg = f"useTrain:{user_id},{arch},{batch_size},{epochs}" + domain_conn.sendall(msg.encode()) + else: + logger.warning(f"No idle domain available for user {user_id} training requests for Arch {arch}") + + + def stop_user_requests(self, user_id: str, arch: str): + if self.domains[list(self.domains.keys())[0]].enable_batching: + if arch in self.model_domain_map: + for domain_id in self.model_domain_map[arch]: + self.domains[domain_id].stop_user_requests(user_id) + else: + for domain_id in self.domains: + self.domains[domain_id].stop_user_requests(user_id) + + def register_domain(self, data, conn): + # message = data.split(":") + message = data.split(":", 1)[1].split(',') + domain_id = message[0] + ip = message[1] + port = int(message[2]) + enable_batching = message[3].lower() == 'true' + # Gateway(domain_id, ip, port, enable_batching) + logger.info(f"Domain {domain_id} registered with IP {ip} and port {port}, enable_batching: {enable_batching}") + self.domains[domain_id] = Domain(domain_id, ip, port, enable_batching, conn) + conn.sendall(f"note:Domain {domain_id} registered successfully.".encode()) + + + + # 处理全局调度器消息 + def handle_glb_message(self, conn, addr): + while True: + data = conn.recv(1024) + if not data: + break + message = data.decode() + if message.startswith('Domain_Regist'): + self.register_domain(message, conn) + elif message.startswith('User_Request_Inference'): + self.start_user_requests(message) + elif message.startswith('User_Reqeust_Training'): + self.start_Training_requests(message) + elif message.startswith('Domainstop'): + message = message.split(":") + domain_id = message[1] + logger.info(f"Domain {domain_id} stopped.") + if domain_id in self.domains: + del self.domains[domain_id] + logger.info(f"Domain {domain_id} removed from scheduler.") + +# 模拟用户请求 +def simulate_user_requests(scheduler: GlobalScheduler): + user_requests = [ + # ("image_data_1", "user1", "resnet18", 5.0, True, 10, 4), # batch_size = 4 + # ("image_data_2", "user2", "resnet18", 3.0, False, 15, 8), # batch_size = 8 + # ("image_data_3", "user3", "vgg16", 4.0, True, 8, 6), + # ("image_data_4", "user4", "resnet18", 6.0, False, 12, 2), + # ("image_data_5", "user5", "densenet121", 2.0, True, 10, 5), + # ("image_data_6", "user6", "resnet18", 7.0, False, 15, 3), + ("image_data_1", "user1", "resnet18", 50, True, 100, 4), + # ("image_data_2", "user2", "resnet18", 50, True, 100, 8), + ("image_data_3", "user3", "vgg16", 50, True, 100, 2), + # ("image_data_4", "user4", "resnet18", 50, True, 100, 4), + ("image_data_5", "user5", "densenet121", 50, True, 100, 1), + # ("image_data_6", "user6", "resnet18", 50, True, 50, 2), + # ("image_data_7", "user7", "resnet18", 40, True, 100, 4), + # ("image_data_8", "user8", "vgg16", 100, True, 100, 4), + # ("image_data_9", "user9", "resnet18", 50, True, 100, 8), + # ("image_data_10", "user10", "densenet121", 6.0, False, 100, 2), + # ("image_data_11", "user11", "resnet50", 50, True, 100, 4), + # ("image_data_12", "user12", "resnet50", 30, True, 120, 4), + # ("image_data_13", "user13", "resnet50", 100, True, 100, 4), + # ("image_data_14", "user14", "resnet50", 50, False, 100, 4), + # ("image_data_15", "user15", "resnet50", 10, True, 100, 4), + # ("image_data_16", "user16", "resnet50", 20, False, 100, 8), + # ("image_data_17", "user17", "resnet50", 100, True, 100, 4), + # ("image_data_18", "user18", "resnet50", 10, False, 100, 5), + ] + + threads = [] + for raw_input, user_id, arch, rps, uniform, num_requests, batch_size in user_requests: + t = threading.Thread(target=lambda: scheduler.start_user_requests(user_id, arch, rps, uniform, num_requests, batch_size)) + threads.append(t) + t.start() + time.sleep(0.05) + + for t in threads: + t.join() + + model_keys = [] + + for raw_input, user_id, arch, rps, uniform, num_requests, batch_size in user_requests: + model_key = f"{arch}_{user_id}" + model_keys.append(model_key) + return model_keys + + + + + +def main(): + config = get_config_file() + glb_Ip = config["GlobalConfig"]["glbIp_"] + glb_Port = config["GlobalConfig"]["glbPort_"] + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind((glb_Ip, glb_Port)) + s.listen() + logging.info("global Scheduler started listen at %s:%d", glb_Ip, glb_Port) + thread_List = [] + + glb_scheduler = GlobalScheduler([], slo_latency=0.5) + + try: + while True: + conn, addr = s.accept() + logging.info("Connected by %s", addr) + thread = threading.Thread(target=glb_scheduler.handle_glb_message, args=(conn, addr)) + thread.start() + thread_List.append(thread) + except KeyboardInterrupt: + logging.info("Server is shutting down...") + s.close() + for thread in thread_List: + thread.join() + logging.info("All threads have been terminated.") + glb_scheduler.stop_event.set() + #TODO: 关闭所有域 + # for domain in glb_scheduler.domains.values(): + # domain.stop() + logging.info("All domains have been stopped.") + +# 主函数 +if __name__ == "__main__": + main() + # # 测试批处理模式 + # domains_batching = [ + # Gateway(domain_id="domain1", ip="10.26.42.231", port=42231, enable_batching=True), + # Gateway(domain_id="domain2", ip="10.26.42.232", port=42232, enable_batching=True), + # Gateway(domain_id="domain3", ip="10.26.42.226", port=22, enable_batching=True), + # ] + # scheduler_batching = GlobalScheduler(domains_batching, slo_latency=0.5) + # logger.info("Starting Global Scheduler with batching...") + # simulate_user_requests(scheduler_batching) + # time.sleep(15) + # for domain in domains_batching: + # for arch in ["resnet18", "vgg16", "densenet121"]: + # throughput = domain.get_throughput(arch) + # if throughput > 0: + # logger.info(f"+++Domain {domain.domain_id} - Arch {arch} Avg Throughput: {throughput:.2f} samples/sec") + + # for domain in domains_batching: + # domain.stop() + # logger.info("Batching simulation completed.") + + # 测试非批处理模式 + # domains_no_batching = [ + # Gateway(domain_id="domain1", ip="10.26.42.231", port=42231, enable_batching=False), + # Gateway(domain_id="domain2", ip="10.26.42.232", port=42232, enable_batching=False), + # Gateway(domain_id="domain3", ip="10.26.42.226", port=22, enable_batching=False), + # ] + # scheduler_no_batching = GlobalScheduler(domains_no_batching, slo_latency=0.5) + # logger.info("Starting Global Scheduler without batching...") + # model_keys = simulate_user_requests(scheduler_no_batching) + # time.sleep(15) + # # for domain in domains_no_batching: + # # for arch in ["resnet18", "vgg16", "densenet121"]: + # # throughput = domain.get_throughput(arch) + # # if throughput > 0: + # # logger.info(f"+++Domain {domain.domain_id} - Arch {arch} Avg Throughput: {throughput:.2f} samples/sec") + # for domain in domains_no_batching: + # for model_key in model_keys: + # throughput = domain.get_throughput(model_key) + # if throughput > 0: + # logger.info(f"+++ Domain {domain.domain_id} arch {model_key}, Avg Throughput: {throughput:.2f} samples/sec") + + + # for domain in domains_no_batching: + # domain.stop() + # logger.info("Non-batching simulation completed.") \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/gpu_info.py b/GPU-Virtual-Service/gpu-remoting/scheduler/gpu_info.py new file mode 100644 index 0000000..4bd901d --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/gpu_info.py @@ -0,0 +1,101 @@ + +import argparse +import socket +import threading +import json +import redis +from ctypes import Structure, c_char, c_int, c_size_t, sizeof +from cffi import FFI +import struct +import logging +from collections import deque +import time +from multiprocessing import Process +import os +import sys + +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.append(parent_dir) + +from scheduler.util import * +from scheduler.job import * + + + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + +class GPU_info: + def __init__(self, gpu_info_properties): + self.gpu_info_properties = gpu_info_properties + gpu_infos = json.loads(gpu_info_properties[b'gpu_info']) + self.gpu_id = gpu_infos['gpu_id'] + self.IP_addr = gpu_infos['IP_addr'] + self.Port = gpu_infos['Port'] + self.memory_total = gpu_infos['memory_total'] + self.memory_free = gpu_infos['memory_free'] + self.memory_used = gpu_infos['memory_used'] + self.utilization = gpu_infos['utilization'] + self.Job_num = gpu_infos['Job_num'] + self.GPU_num = gpu_infos['GPU_num'] + self.HandlerIp = gpu_infos['HandlerIp'] + self.HandlerPort = gpu_infos['HandlerPort'] + self.gpu_properties = gpu_info_properties[b'gpu_properties'] + self.job_list = [] + self.has_high_priority = False + + def gpu_is_free(self, type = 0): + if type == 0: + return self.Job_num < 4 + elif type == 1: + return self.memory_free > self.memory_total * (1 - 0.92) + elif type == 2: + return self.utilization < 90 + elif type == 3: + return self.Job_num == 0 + elif type == 4: + # return self.memory_used < self.memory_total * 0.92 and self.utilization < 95 and self.Job_num < 4 + return self.memory_used < self.memory_total * 0.92 and self.Job_num < 4 + elif type == 5: + return self.memory_free > self.memory_total * 0.92 and self.Job_num < 2 + + def update_gpu_info(self, job, type = 0):#type=0:添加该任务到GPU，type=1:从GPU中删除该任务 + if type == 0: + self.memory_free -= job.gpu_mem + self.memory_used += job.gpu_mem + self.utilization += job.gpu_util + self.Job_num += 1 + self.job_list.append(job) + else: + self.memory_free += job.gpu_mem + self.memory_used -= job.gpu_mem + self.utilization -= job.gpu_util + self.Job_num -= 1 + if job in self.job_list: + self.job_list.remove(job) + + def try_to_allocate(self, job): + if self.memory_free > job.gpu_mem and self.utilization + job.gpu_util < 100 and self.Job_num < 4: + return True + return False + + + def try_to_allocate_lucid(self, job, shared_policy): + if shared_policy == 0: + if self.memory_free > job.gpu_mem and self.utilization + job.gpu_util < 150 and self.Job_num == 0: + return True + else: + return False + else: + if self.memory_free > job.gpu_mem and self.utilization + job.gpu_util < 150 and self.Job_num < 2: + return True + else: + return False + + def __str__(self): + return (f"gpu_id: {self.gpu_id}, IP_addr: {self.IP_addr}, Port: {self.Port}, memory_total: {self.memory_total}, " + f"memory_free: {self.memory_free}, memory_used: {self.memory_used}, utilization: {self.utilization}, " + f"Job_num: {self.Job_num}, GPU_num: {self.GPU_num}, HandlerIp: {self.HandlerIp}, HandlerPort: {self.HandlerPort}, " + f"job_list: {self.job_list}, has_high_priority: {self.has_high_priority}") + \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/job.py b/GPU-Virtual-Service/gpu-remoting/scheduler/job.py new file mode 100644 index 0000000..4c88f75 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/job.py @@ -0,0 +1,98 @@ +import argparse +import socket +import threading +import json +import redis +from ctypes import Structure, c_char, c_int, c_size_t, sizeof +from cffi import FFI +import struct +import logging +from collections import deque +import time +from multiprocessing import Process +import os +import sys + +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.append(parent_dir) + +from scheduler.gpu_info import * + + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + +#================Job_info================ +class Job: + def __init__(self, clientID, ReqGpuNum, priority, model, type, batchsize, gpu_Mem, gpu_util, runtime, conn, addr): + self.clientID = clientID + self.ReqGpuNum = ReqGpuNum + self.priority = priority + self.model = model + self.type = type + self.batchsize = batchsize + self.isDDp = self.ReqGpuNum > 1 + + #连接 + self.ClientConn = conn + self.ServerConn = None + self.addr = addr + #任务信息 + if ReqGpuNum == 1: + self.gpu_mem = gpu_Mem + else: + self.gpu_mem = gpu_Mem * 0.8 + self.gpu_util = gpu_util + self.predict_time = runtime + self.gpu_ids = [] + self.node_list = [] + #time + self.arrival_time = time.time() + self.is_waiting_time = 0 + self.run_time = 0 + self.job_served_time = 0 + self.last_update_time = time.time() + self.preempted_time = 0 + self.is_running = False + #replay + self.allocate_flag = False + self.ready_replay = False + self.reallocated_flag = False + self.waiting_round = 0 + #Themis + self.exclusive_time = 1 + self.shared_time = 1 + self.is_exclusive = False + #Lucid + self.packing_type = -1 # 0:Tiny, 1:Medium, 2:Jumbo + @property + def exclu_shared_ratio(self): + if self.exclusive_time == 0: + return float('inf') + return self.shared_time / self.exclusive_time + + def update_job_info(self, jobPruntime, gpuMem, gpuUtil, priority): + self.predict_time = jobPruntime + self.gpu_mem = gpuMem + self.gpu_util = gpuUtil + self.priority = priority + self.arrival_time = time.time() + self.last_update_time = time.time() + + @property + def job_served(self): + return self.job_served_time * self.ReqGpuNum + + @property + def lucid_priority(self):#Temporal & Spatial Priority + return self.ReqGpuNum * self.predict_time + + def __str__(self): + return (f"Job(clientID={self.clientID}, ReqGpuNum={self.ReqGpuNum}, priority={self.priority}, " + f"model={self.model}, type={self.type}, batchsize={self.batchsize}, gpu_mem={self.gpu_mem}, " + f"gpu_util={self.gpu_util}, run_time={self.run_time}, gpu_ids={self.gpu_ids}, " + f"node_list={self.node_list}, arrival_time={self.arrival_time}, predict_time={self.predict_time}, " + f"is_waiting_time={self.is_waiting_time}, last_update_time={self.last_update_time}, " + f"preempted_time={self.preempted_time}, is_running={self.is_running}, ready_replay={self.ready_replay}, " + f"reallocated_flag={self.reallocated_flag}, waiting_round={self.waiting_round})") \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/job_info.csv b/GPU-Virtual-Service/gpu-remoting/scheduler/job_info.csv new file mode 100644 index 0000000..99ab740 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/job_info.csv @@ -0,0 +1,22 @@ +model,type,batchsize,gpu_mem,gpu_util,runtime +resnet18,0,32,2473984000,18.667,4310 +resnet18,0,64,2786918400,26.667,4670 +resnet18,0,128,3221225472,36.25,4822 +resnet50,0,32,5780275200,38,6012 +resnet50,0,64,9101639680,42,6710 +resnet50,0,128,13958643712,48,7501 +mobilenet_v2,0,32,2631925760,17,3801 +mobilenet_v2,0,64,3156213760,31.5,4012 +mobilenet_v2,0,128,4278190080,41.2,4255 +shufflenet_v2_x0_5,0,32,1361051648,17.1,2471 +shufflenet_v2_x0_5,0,64,1816133632,20.4,2692 +shufflenet_v2_x0_5,0,128,2990538752,24.1,2831 +vgg16,0,32,6142558208,41,6264 +vgg16,0,64,10787749888,51,7245 +vgg16,0,128,20787749888,57,8264 +BERT,0,32,5689917440,38,104 +BERT,0,64,8790835200,42,120 +BERT,0,128,15393189888,46,140 +gpt,0,2,9384755200,43,269 +gpt,0,4,16588877824,48,273 + diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/msg_queue.py b/GPU-Virtual-Service/gpu-remoting/scheduler/msg_queue.py new file mode 100644 index 0000000..ba94c0a --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/msg_queue.py @@ -0,0 +1,207 @@ +import argparse +import socket +import threading +import json +import redis +from ctypes import Structure, c_char, c_int, c_size_t, sizeof +from cffi import FFI +import struct +import logging +from collections import deque +import time +from multiprocessing import Process + +import sys +import os + +# 将 scheduler 目录添加到 sys.path 中 +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.append(parent_dir) + +from scheduler.cluster import * + +import threading +import time +import random + +# r = redis_connection() + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +# 公共注册频道 +REGISTER_CHANNEL = "channel:register" + +# 模拟客户端类 +class MSG_Client: + def __init__(self, client_id): + self.client_id = client_id + self.redis_conn = redis_connection() + # self.client_channel = client_id # 接收服务器消息 + self.status_channel = f"{self.client_id}_status" # 状态消息频道 + self.data_channel = self.client_id # 数据消息频道 + self.server_channel = f"server_{client_id}" # 发送给服务器 + self.pubsub_status = self.redis_conn.pubsub() # 状态订阅 + self.pubsub_data = self.redis_conn.pubsub() # 数据订阅 + self.pubsub_status.subscribe(self.status_channel) + self.pubsub_data.subscribe(self.data_channel) + # self.pubsub = self.redis_conn.pubsub() + # self.pubsub.subscribe(self.client_channel) + self.running = True + # 客户端启动时注册自己 + self.registered = False # 注册状态 + self.allocated = False + self.register() + + def register(self): + # 发送注册消息到公共频道 + self.redis_conn.publish(REGISTER_CHANNEL, self.client_id) + print(f"Client {self.client_id} registered itself") + + def listen(self): + print(f"Client {self.client_id} started, listening on {self.status_channel}") + while self.running: + message = self.pubsub_status.get_message(timeout=1.0) + if message and message['type'] == 'message': + data = message['data'].decode('utf-8') + if data == "REGISTERED": + self.registered = True + elif data == "ALLOCATED": + self.allocated = True + print(f"Client {self.client_id} received: {message['data'].decode()}") + time.sleep(0.01) + + def send_message(self, message): + timeout = 5 # 最多等待 5 秒 + start_time = time.time() + # while not self.registered and time.time() - start_time < timeout: + while not self.registered: + time.sleep(1) + if not self.registered: + logging.error(f"Client {self.client_id} failed to register within {timeout} seconds") + return + full_message = f"{message}" + self.redis_conn.publish(self.server_channel, full_message) + print(f"Client {self.client_id} sent: {full_message}") + + def stop(self): + self.running = False + # self.pubsub.unsubscribe(self.client_channel) + self.pubsub_status.unsubscribe(self.status_channel) + self.pubsub_data.unsubscribe(self.data_channel) + # self.redis_conn.close() + print(f"Client {self.client_id} stopped") + +# 服务器类 +class MSG_Server: + def __init__(self): + self.clients = {} + self.redis_conn = redis_connection() + self.pubsub = self.redis_conn.pubsub() + self.running = True + # 订阅注册频道 + self.pubsub.subscribe(REGISTER_CHANNEL) + + def register_client(self, client_id): + if client_id not in self.clients: + self.clients[client_id] = { + "status_channel": f"{client_id}_status", + "data_channel": client_id, + "send_channel": f"server_{client_id}" + } + self.pubsub.subscribe(self.clients[client_id]["send_channel"]) + logging.info(f"Server registered client {client_id}") + self.redis_conn.publish(self.clients[client_id]["status_channel"], "REGISTERED") + + def send_message(self, target_client_id, message): + if target_client_id in self.clients: + channel = self.clients[target_client_id]["receive_channel"] + full_message = f"server:{message}" + self.redis_conn.publish(channel, full_message) + print(f"Server sent to {target_client_id}: {full_message}") + + def listen(self): + print(f"Server started, listening for registrations and messages") + while self.running: + message = self.pubsub.get_message(timeout=1.0) + if message and message['type'] == 'message': + channel = message['channel'].decode() + data = message['data'].decode() + if channel == REGISTER_CHANNEL: + # 处理注册消息 + self.register_client(data) + else: + # 处理客户端消息 + client_id = channel.split("_")[-1] + print(f"Server received from {client_id}: {data}") + time.sleep(0.01) + + def send_to_client(self, gpu_info_list, clientID): + # print(f'send to clientID:{clientID}') + logging.info(f'Send to clientID:{clientID}') + client_channel = clientID # 使用 clientID 作为频道 + + # self.msg_server.redis_conn.publish(self.msg_server.clients[client_id]["status_channel"], "ALLOCATED") + self.redis_conn.publish(self.clients[str(client_channel)]["status_channel"], "ALLOCATED") + + if str(clientID) not in self.clients: + # print(f"Client {clientID} not registered yet") + logging.error(f"Client {clientID} not registered yet") + return + print(f'gpu_info_list: {gpu_info_list}') + for gpu_info in gpu_info_list: + if gpu_info is None: + self.redis_conn.publish(self.clients[clientID]["data_channel"], None) + # print(f"Sent None to {clientID}") + logging.info(f"Sent None to {clientID}") + return + avail_gpu = { + 'gpu_id': gpu_info.gpu_id, + 'IP_addr': gpu_info.IP_addr, + 'Port': gpu_info.Port, + 'HandlerIp': gpu_info.HandlerIp, + 'HandlerPort': gpu_info.HandlerPort + } + gpu_properties = gpu_info.gpu_properties + + gpuInfo_bytes = struct.pack( + 'iH40sH40s', + avail_gpu['gpu_id'], + avail_gpu['Port'], + avail_gpu['IP_addr'].encode('utf-8'), + avail_gpu['HandlerPort'], + avail_gpu['HandlerIp'].encode('utf-8') + ) + data_len = len(gpu_properties) + len_bytes = socket.htonl(data_len).to_bytes(8, 'big') + full_message = gpuInfo_bytes + len_bytes + gpu_properties + + + self.redis_conn.publish(client_channel, full_message) + # print(f'data: {full_message}') + # print(f"Sent GPU info to {clientID}: {len(full_message)} bytes") + logging.info(f'Sent GPU info to {clientID}: {len(full_message)} bytes') + + + + def run(self):#dummy test + while self.running: + if self.clients: + target_id = random.choice(list(self.clients.keys())) + self.send_message(target_id, f"Hello at {time.ctime()}") + time.sleep(5) + + def stop_client(self, client_id): + if client_id in self.clients: + self.pubsub.unsubscribe(self.clients[client_id]["send_channel"]) + del self.clients[client_id] + logging.info(f"Sent stop message to client {client_id}") + else: + logging.error(f"Client {client_id} not found") + + def stop(self): + self.running = False + self.pubsub.unsubscribe(REGISTER_CHANNEL) + for client_id in self.clients: + self.pubsub.unsubscribe(self.clients[client_id]["send_channel"]) + # self.redis_conn.close() + print("Server stopped") diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/node.py b/GPU-Virtual-Service/gpu-remoting/scheduler/node.py new file mode 100644 index 0000000..220afb2 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/node.py @@ -0,0 +1,112 @@ +import argparse +import socket +import threading +import json +import redis +from ctypes import Structure, c_char, c_int, c_size_t, sizeof +from cffi import FFI +import struct +import logging +from collections import deque +import time +from multiprocessing import Process +import os +import sys + + +# 将 scheduler 目录添加到 sys.path 中 +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.append(parent_dir) + +#自定义类 +from scheduler.util import * +from scheduler.gpu_info import* +from scheduler.job import * + + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + +class Node: + def __init__(self, id, ip): + self.id = id + self.ip = ip + self.job_run_list = [] + self.job_run_list_lock = threading.Lock() + self.gpu_list = [] + + r = redis_connection() + config = get_config_file() + self.gpu_num = config['ServerConfig']['serverGPU_'] + for i in range(self.gpu_num): + redis_key = f"{self.ip}:{i}" + gpu_info_pro = r.hgetall(redis_key) + gpu = GPU_info(gpu_info_pro) + self.gpu_list.append(gpu) + print(gpu) + + + def update_gpu_info(self, gpu_id, job, type):#type=0:分配内存，type=1:释放内存 + for gpu in self.gpu_list: + if gpu.gpu_id == gpu_id: + cur_gpu = gpu + if cur_gpu != None: + cur_gpu.update_gpu_info(job, type) + + def sort_gpu_list(self, type, rev): + if type == 1: + if rev: + self.gpu_list.sort(key=lambda x: x.Job_num, reverse = True) + else: + self.gpu_list.sort(key=lambda x: x.Job_num) + elif type == 2: + if rev: + self.gpu_list.sort(key=lambda x: x.memory_free, reverse = True) + else: + self.gpu_list.sort(key=lambda x: x.memory_free) + elif type == 3: + if rev: + self.gpu_list.sort(key=lambda x: x.utilization, reverse = True) + else: + self.gpu_list.sort(key=lambda x: x.utilization) + + def has_free_gpu(self, type): + for gpu in self.gpu_list: + if gpu.gpu_is_free(type): + return True + return False + + def sort_job_list(self, type): + if type == 1: + self.job_run_list.sort(key=lambda x: x.arrival_time) + elif type == 2: + self.job_run_list.sort(key=lambda x: x.runtime, reverse = True) + elif type == 3: + self.job_run_list.sort(key=lambda x: x.job_served, reverse = True) + + + def free_gpu_num(self, type): + count = 0 + for gpu in self.gpu_list: + if gpu.gpu_is_free(type): + count += 1 + return count + + def free_gpus(self, type): + free_gpu_list = [] + for gpu in self.gpu_list: + if gpu.gpu_is_free(type): + free_gpu_list.append(gpu) + return free_gpu_list + + @property + def total_job_num(self): + return len(self.job_run_list) + + def has_free_gpu_num(self, type): + count = 0 + for gpu in self.gpu_list: + if gpu.gpu_is_free(type): + count += 1 + return count \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/requese_handler.py b/GPU-Virtual-Service/gpu-remoting/scheduler/requese_handler.py new file mode 100644 index 0000000..714ab0f --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/requese_handler.py @@ -0,0 +1,635 @@ +import os +import sys +import torch +import torch.nn as nn +from torchvision import models, transforms +from collections import deque +import time +import threading +import logging +# from typing import Dict, List, Any +from typing import Dict, List, Any, Optional +import numpy as np +import uuid +import socket +import signal +import subprocess +from concurrent.futures import ThreadPoolExecutor +#Version ： +# 将 scheduler 目录添加到 sys.path 中 +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.append(parent_dir) + +#自定义类 +from scheduler.util import * +from scheduler.gpu_info import* +from scheduler.job import * +from scheduler.node import * + + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + +# 设置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) +# 输入预处理 +preprocess = transforms.Compose([ + transforms.Resize((224, 224)), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) +]) + +# 为张量添加时间戳和批次大小的类 +class TimestampedTensor: + def __init__(self, tensor: torch.Tensor, batch_size: int): + self.tensor = tensor + self.timestamp = time.time() + self.batch_size = batch_size + +class ModelManager: + def __init__(self, num_gpus: int = 2): + self.models: Dict[str, nn.Module] = {} + self.gpu_assignments: Dict[str, int] = {} # 模型 key 到 GPU ID 的映射 + self.lock = threading.Lock() + self.num_gpus = num_gpus # 可用 GPU 数量 + logger.info(f"ModelManager initialized with {self.num_gpus} GPUs") + + def get_or_create_model(self, key: str, enable_batching) -> nn.Module: + with self.lock: + if key in self.models: + logger.info(f"Reusing existing model: {key} on GPU {self.gpu_assignments[key]}") + return self.models[key] + try: + if not enable_batching: + arch = key.split('_')[0] # 假设 key 是 "arch_userid" 格式 + else: + arch = key + # 根据 key 分配 GPU（简单哈希方式） + gpu_id = hash(key) % self.num_gpus if self.num_gpus > 0 else 0 + device = torch.device(f"cuda:{gpu_id}" if torch.cuda.is_available() and self.num_gpus > 0 else "cpu") + + model_fn = getattr(models, arch, None) + if model_fn is None or not callable(model_fn): + raise ValueError(f"Invalid architecture: {arch}") + logger.info(f"Choosing device: {device} for model {key}") + model = model_fn(pretrained=True).to(device) + model.eval() + self.models[key] = model + self.gpu_assignments[key] = gpu_id # 记录分配的 GPU ID + logger.info(f"Created new model: {key} on GPU {gpu_id}") + return model + except Exception as e: + logger.error(f"Failed to load model {key}: {e}") + raise + + def remove_model(self, key: str): + with self.lock: + if key in self.models: + del self.models[key] + del self.gpu_assignments[key] + logger.info(f"Removed model: {key}") + +# Gateway 类 +class Gateway: + def __init__(self, domain_id, ip, port, glbconn, timeout: float = 0.2, max_latency: float = 2.0, enable_batching: bool = True, max_concurrent_requests: int = 10): + self.domain_id = domain_id + self.region_ip = ip + self.region_port = port + self.glbconn = glbconn + # self.model_manager = ModelManager() + self.num_gpus = torch.cuda.device_count() if torch.cuda.is_available() else 0 + self.model_manager = ModelManager(num_gpus=self.num_gpus) + self.buffers: Dict[str, deque] = {} + self.request_maps: Dict[str, Dict[str, Tuple[str, int]]] = {} + self.timeout = timeout + self.max_latency = max_latency + self.locks: Dict[str, threading.Lock] = {} + self.results: Dict[str, torch.Tensor] = {} + self.running: Dict[str, bool] = {} + self.threads: Dict[str, threading.Thread] = {} + self.latency: Dict[str, float] = {} + self.latency_history: Dict[str, deque] = {} + self.first_inference: Dict[str, bool] = {} + self.last_request_time: Dict[str, float] = {} + self.enable_batching = enable_batching #是否开启Batching + self.max_concurrent_requests = max_concurrent_requests + self.active_requests: Dict[str, int] = {} # 每个用户的活跃请求数 + self.request_semaphore: Dict[str, threading.Semaphore] = {} # 每个用户的并发控制 + self.user_models: Dict[str, str] = {} # 用户到模型 key 的映射 + self.model_keys = [] + # 吞吐量统计 + self.throughput_history: Dict[str, deque] = {} + + # 多用户请求生成相关 + self.user_threads: Dict[str, threading.Thread] = {} + self.user_configs: Dict[str, dict] = {} + self.user_sleep_times: Dict[str, list] = {} + self.user_lock = threading.Lock() + self.user_running: Dict[str, bool] = {} + self.thread_pool = ThreadPoolExecutor(max_workers=20) + + # 非批处理模式下的请求线程 + self.request_threads: Dict[str, threading.Thread] = {} + + # GPU 资源信息 + self.stop_flag = False + config = get_config_file() + local_ip = config['RequestConfig']['localIp_'] + r = redis_connection() + self.gpu_list = [] + for i in range(2): + redis_key = f"{local_ip}:{i}" + gpu_info_pro = r.hgetall(redis_key) + gpu = GPU_info(gpu_info_pro) + self.gpu_list.append(gpu) + logger.info(f"GPU {i} info: {gpu}") + + #处理Training请求 + self.train_model_gpus: Dict[str, int] = {} # 模型到 GPU 的映射 + + if self.enable_batching: + self.idle_check_thread = threading.Thread(target=self._check_idle_models, daemon=True) + self.idle_check_thread.start() + + def _generate_sleep_times(self, rps: float, uniform: bool, num_requests: int) -> list: + if rps > 0: + if uniform: + return [1/rps] * num_requests + else: + return np.random.exponential(scale=1/rps, size=num_requests).tolist() + return [0] * num_requests + + def _get_model_resources(self, arch: str) -> tuple: + if arch not in self.buffers: + self.buffers[arch] = deque() + self.request_maps[arch] = {} + self.locks[arch] = threading.Lock() + self.running[arch] = True + self.latency[arch] = 0.0 + self.latency_history[arch] = deque(maxlen=50) + self.first_inference[arch] = True + self.last_request_time[arch] = time.time() + self.active_requests[arch] = 0 + self.request_semaphore[arch] = threading.Semaphore(self.max_concurrent_requests) + self.threads[arch] = threading.Thread(target=self._check_and_process_batch, args=(arch,), daemon=True) + self.threads[arch].start() + return self.buffers[arch], self.request_maps[arch], self.locks[arch] + + def preprocess_input(self, raw_input: Any, batch_size: int,) -> TimestampedTensor: + try: + input_tensor = torch.randn(batch_size, 3, 224, 224) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + return TimestampedTensor(input_tensor.to(device), batch_size) + except Exception as e: + logger.error(f"Preprocess error: {e}") + raise + + def process_batch(self, model: nn.Module, batch: torch.Tensor, arch: str) -> tuple: + try: + with torch.no_grad(): + # if torch.cuda.is_available(): + # batch = batch.cuda() + # start_time = time.time() + # outputs = model(batch) + # latency = time.time() - start_time + # throughput = batch.size(0) / latency + + if torch.cuda.is_available(): + gpu_id = self.model_manager.gpu_assignments[arch] # 获取模型的 GPU ID + batch = batch.to(f"cuda:{gpu_id}") + start_time = time.time() + outputs = model(batch) + latency = time.time() - start_time + throughput = batch.size(0) / latency + + if arch in self.throughput_history: + self.throughput_history[arch].append(throughput) + avg_throughput = np.mean(list(self.throughput_history[arch])) + else: + # self.throughput_history[arch] = deque(maxlen=50) + self.throughput_history[arch] = deque() + + avg_throughput = throughput + + # with self.locks[arch]: + if self.first_inference[arch]: + logger.info(f"Cold start for Arch {arch}, latency {latency:.4f} not counted") + self.first_inference[arch] = False + else: + self.latency_history[arch].append(latency) + logger.info(f"_+_+_+!!Arch {arch} latency history: {self.latency_history[arch]}") + latencies = list(self.latency_history[arch]) + logger.info(f"_+_+_+!!Arch {arch} latencies: {latencies}") + if latencies: + logger.info(f"_+_+_+!!Arch {arch} latencies: {latencies}") + self.latency[arch] = np.percentile(latencies, 90) + logger.info(f"_+_+_+!!Arch {arch} latency: {self.latency[arch]}") + else: + self.latency[arch] = 0.0 + + + logger.info(f"Arch {arch}, Batch_size {batch.size(0)}, Throughput: {throughput:.2f} samples/sec, Latency: {latency:.4f} seconds, avg Throughput: {avg_throughput}, P90 Latency: {self.latency[arch]:.4f}") + return outputs, latency + except Exception as e: + logger.error(f"Batch processing error: {e}") + raise + + def process_single_request(self, raw_input: Any, user_id: str, arch: str, request_id: str, batch_size: int): + """非批处理模式下处理用户请求，使用用户绑定的模型""" + model_key = f"{arch}_{user_id}" + self.latency_history[model_key] = deque(maxlen=50) + if model_key not in self.throughput_history: + # self.throughput_history[model_key] = deque(maxlen=50) + self.throughput_history[model_key] = deque() + self.model_keys.append(model_key) + + if user_id not in self.active_requests: + self.active_requests[user_id] = 0 + if user_id not in self.request_semaphore: + self.request_semaphore[user_id] = threading.Semaphore(self.max_concurrent_requests) + + with self.request_semaphore[user_id]: + with self.user_lock: + self.active_requests[user_id] += 1 + if user_id not in self.user_models: + self.user_models[user_id] = model_key + logger.info(f"_+_+_+Processing single request {request_id} for User {user_id} on Arch {arch}, Batch Size: {batch_size}") + model = self.model_manager.get_or_create_model(model_key, False) + input_tensor = self.preprocess_input(raw_input, batch_size) + try: + # with torch.no_grad(): + # if torch.cuda.is_available(): + # input_tensor.tensor = input_tensor.tensor.cuda() + # start_time = time.time() + # output = model(input_tensor.tensor) + # latency = time.time() - start_time + # throughput = batch_size / latency + with torch.no_grad(): + if torch.cuda.is_available(): + gpu_id = self.model_manager.gpu_assignments[model_key] # 获取模型的 GPU ID + # input_tensor.tensor = input_tensor.tensor.to(f"cuda:{gpu_id}") + batch = input_tensor.tensor.to(f"cuda:{gpu_id}") + start_time = time.time() + output = model(batch) + latency = time.time() - start_time + throughput = batch_size / latency + + with self.user_lock: + if self.first_inference.get(model_key, True): + avg_throughput = throughput + self.first_inference[model_key] = False + else: + self.latency_history[model_key].append(latency) + latencies = list(self.latency_history[model_key]) + if latencies: + self.latency[model_key] = np.percentile(latencies, 90) + else: + self.latency[model_key] = 0.0 + self.throughput_history[model_key].append(throughput) + avg_throughput = np.mean(list(self.throughput_history[model_key])) + self.results[request_id] = output + self.active_requests[user_id] -= 1 + + logger.info(f"Processed single request for User {user_id} on Arch {arch}, Batch Size: {batch_size}, Latency: {latency:.4f} seconds, Throughput: {throughput:.2f} samples/sec, Avg Throughput: {avg_throughput:.2f} samples/sec") + except Exception as e: + logger.error(f"Single request processing failed: {e}") + with self.user_lock: + self.active_requests[user_id] -= 1 + + def send_result(self, arch: str, request_id: str, result: torch.Tensor): + user_id, _ = self.request_maps[arch].get(request_id, (None, None)) + self.results[request_id] = result + logger.info(f"Result sent to User {user_id} for Request {request_id} (Arch: {arch}): {result.shape}") + + def handle_request(self, raw_input: Any, user_id: str, arch: str, batch_size: int) -> str: + request_id = str(uuid.uuid4()) + if self.enable_batching: + buffer, request_map, lock = self._get_model_resources(arch) + with lock: + input_tensor = self.preprocess_input(raw_input, batch_size) + buffer.append(input_tensor) + request_map[request_id] = (user_id, batch_size) + self.last_request_time[arch] = time.time() + logger.info(f"Received request {request_id} from User {user_id} for Arch {arch}, Batch Size: {batch_size}, Buffer size: {len(buffer)}") + else: + # thread = threading.Thread(target=self.process_single_request, args=(raw_input, user_id, arch, request_id, batch_size), daemon=True) + # self.request_threads[request_id] = thread + # thread.start() + self.thread_pool.submit(self.process_single_request, raw_input, user_id, arch, request_id, batch_size) + logger.info(f"Received and processing request {request_id} from User {user_id} for Arch {arch}, Batch Size: {batch_size} (no batching)") + return request_id + + def _check_and_process_batch(self, arch: str): + buffer, request_map, lock = self._get_model_resources(arch) + + while self.running.get(arch, False): + with lock: + if buffer: + total_batch_size = sum(item.batch_size for item in buffer) + if total_batch_size > 0 and (len(buffer) >= 1 and (time.time() - buffer[0].timestamp > self.timeout or total_batch_size >= max(item.batch_size for item in buffer))): + batch_list = list(buffer) + batch = torch.cat([item.tensor for item in batch_list], dim=0) + logger.info(f"Processing batch of size {batch.size(0)} for Arch {arch}") + model = self.model_manager.get_or_create_model(arch, True) + try: + results, latency = self.process_batch(model, batch, arch) + offset = 0 + request_ids = list(request_map.keys())[:len(batch_list)] + for i, req_id in enumerate(request_ids): + batch_size = batch_list[i].batch_size + self.send_result(arch, req_id, results[offset:offset + batch_size]) + offset += batch_size + del request_map[req_id] + buffer.clear() + except Exception as e: + logger.error(f"Batch processing failed for {arch}: {e}") + time.sleep(0.01) + + def _check_idle_models(self): + while True: + with self.user_lock: + current_time = time.time() + for arch in list(self.running.keys()): + last_time = self.last_request_time.get(arch, current_time) + if current_time - last_time > 10 and self.running.get(arch, False): + logger.info(f"Model {arch} in Domain {self.domain_id} idle for 10s, shutting down") + self.running[arch] = False + if arch in self.threads and self.threads[arch].is_alive(): + self.threads[arch].join() + if arch in self.buffers: + del self.buffers[arch] + if arch in self.request_maps: + del self.request_maps[arch] + if arch in self.locks: + del self.locks[arch] + # if arch in self.latency: + # del self.latency[arch] + # if arch in self.latency_history: + # del self.latency_history[arch] + if arch in self.threads: + del self.threads[arch] + time.sleep(1) + + def _generate_requests(self, user_id: str, arch: str, batch_size: int): + config = self.user_configs[user_id] + sleep_times = self.user_sleep_times[user_id] + for i in range(config['num_requests']): + if not self.user_running.get(user_id, False): + break + raw_input = None + self.handle_request(raw_input, user_id, arch, batch_size) + sleep_time = sleep_times[i] + logger.info(f"User {user_id} - Generated request {i+1}/{config['num_requests']}, Batch Size: {batch_size}, sleeping for {sleep_time:.4f}s") + time.sleep(sleep_time) + + def start_user_requests(self, user_id: str, arch: str, rps: float, uniform: bool, num_requests: int, batch_size: int): + with self.user_lock: + if user_id in self.user_threads and self.user_threads[user_id].is_alive(): + logger.info(f"User {user_id} is already running") + return + + self.user_configs[user_id] = { + 'rps': rps, + 'uniform': uniform, + 'num_requests': num_requests, + 'batch_size': batch_size + } + self.user_sleep_times[user_id] = self._generate_sleep_times(rps, uniform, num_requests) + self.user_running[user_id] = True + + thread = threading.Thread( + target=self._generate_requests, args=(user_id, arch, batch_size), daemon=True + ) + self.user_threads[user_id] = thread + thread.start() + logger.info(f"Started request generator for User {user_id} on Arch {arch} with {'uniform' if uniform else 'poisson'} distribution, Batch Size: {batch_size}") + + def stop_user_requests(self, user_id: str): + with self.user_lock: + if user_id in self.user_running: + self.user_running[user_id] = False + if user_id in self.user_threads and self.user_threads[user_id].is_alive(): + self.user_threads[user_id].join() + if user_id in self.user_models: + self.model_manager.remove_model(self.user_models[user_id]) + del self.user_models[user_id] + if user_id in self.active_requests: + del self.active_requests[user_id] + if user_id in self.request_semaphore: + del self.request_semaphore[user_id] + logger.info(f"Stopped request generator for User {user_id}") + + def get_result(self, request_id: str) -> Optional[torch.Tensor]: + return self.results.get(request_id, None) + + def stop(self): + for arch in self.running: + self.running[arch] = False + for thread in self.threads.values(): + thread.join() + self.stop_flag = True + + with self.user_lock: + for user_id in self.user_running: + self.user_running[user_id] = False + if user_id in self.user_threads and self.user_threads[user_id].is_alive(): + self.user_threads[user_id].join() + if user_id in self.user_models: + self.model_manager.remove_model(self.user_models[user_id]) + del self.user_models[user_id] + for thread in self.request_threads.values(): + if thread.is_alive(): + thread.join() + logger.info(f"Domain {self.domain_id} stopped") + + def get_load(self, arch: str) -> float: + return self.latency.get(arch, 0.0) if self.enable_batching else 0.0 + + def get_throughput(self, arch: str) -> float: + # logger.info(f"domain id {self.domain_id} get throughput {arch}, len {len(self.throughput_history)}") + # logger.info(f"Throughput history keys: {list(self.throughput_history.keys())}") + if arch in self.throughput_history and self.throughput_history[arch]: + return np.mean(list(self.throughput_history[arch])) + return 0.0 + + def get_latency(self, arch: str) -> float: + # logger.info(f"_+_+!!domain id {self.domain_id} get latency {arch},") + # logger.info(f"_+_+!!domain id {self.domain_id} get latency {arch}, ") + + if arch in self.latency: + return self.latency[arch] + return 0.0 + + def handle_client_request(self, message): + data = message.split(":", 1)[1].split(',') + user_id = data[0] + arch = data[1] + rps = float(data[2]) + # uniform = bool(int(data[3])) + uniform = data[3].lower() == 'true' + num_requests = int(data[4]) + batch_size = int(data[5]) + logger.info(f"Received user request: {message}") + if self.enable_batching: + logger.info(f"Enable_Batching Starting user requests for User {user_id} on Arch {arch}, RPS: {rps}, Uniform: {uniform}, Num Requests: {num_requests}, Batch Size: {batch_size}") + self.start_user_requests(user_id, arch, rps, uniform, num_requests, batch_size) + else: + logger.info(f"Non-Batching Starting user requests for User {user_id} on Arch {arch}, Batch Size: {batch_size}") + self.start_user_requests(user_id, arch, rps, uniform, num_requests, batch_size) + + def handle_train_request(self, message): + data = message.split(":", 1)[1].split(',') + user_id = data[0] + arch = data[1] + batch_size = int(data[2]) + num_epochs = int(data[3]) + #python scripts/workloads/imageNetTrain.py -a arch -b batch_size -e num_epochs --first_batch --gpu gpu + logger.info(f"Received user training request: {message}") + # 处理用户训练请求 + self.gpu_list.sort(key=lambda x: x.memory_free, reverse=True) + gpu_id = self.gpu_list[0].gpu_id + self.gpu_list[0].memory_free -= 3000 * 2**20 + self.train_model_gpus[arch] = gpu_id + # command = [ + # "python", "scripts/workloads/imageNetTrain.py", + # "-a", arch, + # "-b", str(batch_size), + # "-e", str(num_epochs), + # "--first_batch", + # "--gpu", str(gpu_id) + # ] + command = f"python scripts/workloads/imageNetTrain_.py -a {arch} -b {batch_size} --epochs {num_epochs} --first_batch --gpu {gpu_id} --batch_rate 0.1" + logger.info(f"Executing command: {command}") + subprocess.Popen(command, shell=True) + + #异步接受GlbobalServer的消息 + def receive_messages(self,): + start_time = time.time() + while True: + try: + data = self.glbconn.recv(1024) + if not data: + break + message = data.decode() + logger.info(f"Received message from Global Server: {message}") + if message.startswith("usereq:"): + logger.info(f"Received user request: {message}") + self.handle_client_request(message) + elif message.startswith("useTrain:"): + logger.info(f"Received user training request: {message}") + # 处理用户训练请求 + self.handle_train_request(message) + elif message == "stop": + self.stop() + break + elif message.startswith("note:"): + print(message) + # 处理消息 + except KeyboardInterrupt as e: + # logger.error(f"Error receiving message: {e}") + self.stop() + break + + +def signal_handler(sig, frame, glb_socket, domain_id, gateway): + """处理 Ctrl+C 信号，发送 Domainstop 消息并关闭 socket""" + logger.info("Received Ctrl+C, sending Domainstop...") + model_total_throughput = {} + model_total_latency = {} + try: + if gateway.enable_batching: + for arch in list(gateway.running.keys()): + logger.info(f"=============== {arch} in Domain {domain_id} ===============") + throughput = gateway.get_throughput(arch) + latency = gateway.get_latency(arch) + if latency > 0: + logger.info(f"Model< {arch} > in Domain {domain_id} latency: {latency:.4f}") + else: + logger.info(f"Model< {arch} > in Domain {domain_id} latency: 0.0") + if throughput > 0: + logger.info(f"Model< {arch} > in Domain {domain_id} throughput: {throughput:.2f}") + else: + for model_key in list(gateway.model_keys): + logger.info(f"=============== {model_key} in Domain {domain_id} ===============") + throughput = gateway.get_throughput(model_key) + latency = gateway.get_latency(model_key) + if latency > 0: + logger.info(f"Model< {model_key} > in Domain {domain_id} latency: {latency:.4f}") + model, user = model_key.split('_') + if model not in model_total_latency: + model_total_latency[model] = [] + model_total_latency[model].append(latency) + else: + logger.info(f"Model< {model_key} > in Domain {domain_id} latency: 0.0") + if throughput > 0: + logger.info(f"Model< {model_key} > in Domain {domain_id} throughput: {throughput:.2f}") + model, user = model_key.split('_') + if model not in model_total_throughput: + model_total_throughput[model] = [] + model_total_throughput[model].append(throughput) + + if not gateway.enable_batching: + # 计算平均吞吐量和延迟 + logger.info(f"\n================= no batching {domain_id} Summary =================") + for model in model_total_latency.keys(): # 遍历所有模型 + # 计算平均 latency + avg_latency = np.mean(model_total_latency[model]) + # 计算平均 throughput + avg_throughput = np.mean(model_total_throughput.get(model, [])) # 防止 KeyError + # 输出模型信息（latency + throughput） + logger.info( + f"Model< {model} > in Domain {domain_id} | " + f"Avg Latency: {avg_latency:.4f} | " + f"Avg Throughput: {avg_throughput:.2f}" + ) + stop_msg = f"Domainstop:{domain_id}" + glb_socket.sendall(stop_msg.encode()) + time.sleep(0.5) # 确保消息发送完成 + except Exception as e: + logger.error(f"Failed to send Domainstop: {e}") + finally: + glb_socket.close() + sys.exit(0) # 退出程序 + +def main(): + config = get_config_file() + domain_id = config['RequestConfig']['domain_id_'] + domain_ip = config['RequestConfig']['ReqIp_'] + domain_port = config['RequestConfig']['ReqPort_'] + domian_enable_batching = config['RequestConfig']['enable_batching_'] + + # gateway = Gateway(domain_id, domain_ip, domain_port, enable_batching=True) + #注册GateWay + glb_ip = config['GlobalConfig']['glbIp_'] + glb_port = config['GlobalConfig']['glbPort_'] + glb_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + glb_socket.connect((glb_ip, glb_port)) + register_msg = f"Domain_Regist:{domain_id},{domain_ip},{domain_port},{domian_enable_batching}" + glb_socket.sendall(register_msg.encode()) + + # 设置 Ctrl+C 信号处理 + + gateway = Gateway(domain_id, domain_ip, domain_port, glb_socket, enable_batching=domian_enable_batching) + + signal.signal(signal.SIGINT, lambda sig, frame: signal_handler(sig, frame, glb_socket, domain_id, gateway)) + + + receive_thread = threading.Thread(target=gateway.receive_messages, daemon=True) + receive_thread.start() + + while not gateway.stop_flag: + time.sleep(1) + + try: + while not gateway.stop_flag: + time.sleep(1) + except KeyboardInterrupt: + pass # 由 signal_handler 处理 + finally: + glb_socket.close() + logger.info("Gateway stopped") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/resource_scheduler.py b/GPU-Virtual-Service/gpu-remoting/scheduler/resource_scheduler.py new file mode 100644 index 0000000..2d1dad4 --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/resource_scheduler.py @@ -0,0 +1,694 @@ +import argparse +import socket +import threading +import json +import redis +from ctypes import Structure, c_char, c_int, c_size_t, sizeof +from cffi import FFI +import struct +import logging +from collections import deque +import time +from multiprocessing import Process +import os +import sys +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime + +# 将 scheduler 目录添加到 sys.path 中 +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.append(parent_dir) + +#自定义类 +from scheduler.util import * +from scheduler.gpu_info import* +from scheduler.job import * +from scheduler.node import * +from scheduler.msg_queue import * + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + + +class Scheduler: + def __init__(self): + # self.cluster_list = [] + # self.cluster_list_lock = threading.Lock() + self.node_list = [] + self.node_list_lock = threading.Lock() + self.all_job_list = [] + self.all_job_list_lock = threading.Lock() + self.allocated_list = [] + self.preemt_job_list = [] + self.preemt_job_list_lock = threading.Lock() + self.is_running_list = [] + self.is_running_list_lock = threading.Lock() + self.waiting_job_list = [] + self.waiting_job_list_lock = threading.Lock() + self.msg_server = MSG_Server() + self.executor = ThreadPoolExecutor(max_workers=5) + self.scheduler_running = False + #scheduler_Queue + self.highpriority_queue = [] + self.highpriority_queue_lock = threading.Lock() + self.lowpriority_queue = [] + self.lowpriority_queue_lock = threading.Lock() + self.Threshold = 400 + self.scheduler_time_interval = 20 + self.stop_event = threading.Event() + self.logging_level = 0 #0:INFO, 1:DEBUG + self.finish_flag = 0 + + def print_gpu_info(self): + for node in self.node_list: + for gpu in node.gpu_list: + logging.info(f"GPU {gpu.IP_addr}:{gpu.gpu_id} memory:{gpu.memory_free} util:{gpu.utilization}") + + + + def sort_node_list(self, type, free_gpu_type, rev): + if type == 1:#按照资源最空闲的节点排序 + if rev: + self.node_list.sort(key=lambda x: x.free_gpu_num(free_gpu_type), reverse = True) + else: + self.node_list.sort(key=lambda x: x.free_gpu_num(free_gpu_type)) + elif type == 2:#按照job的数量排序 + if rev: + self.node_list.sort(key=lambda x: x.total_job_num, reverse = True) + else: + self.node_list.sort(key=lambda x: x.total_job_num) + + def has_free_resource(self): + for node in self.node_list: + if node.has_free_gpu(0): + return True + return False + + def clean_client_job(self, client_id):#client自己出错的情况，清理所有client资源 + cur_job = None + for job in self.all_job_list: + if job.clientID == client_id: + cur_job = job + break + if cur_job == None: + return + if cur_job in self.all_job_list: + self.all_job_list.remove(cur_job) + if cur_job in self.highpriority_queue: + self.highpriority_queue.remove(cur_job) + if cur_job in self.lowpriority_queue: + self.lowpriority_queue.remove(cur_job) + if cur_job in self.preemt_job_list: + self.preemt_job_list.remove(cur_job) + if cur_job in self.is_running_list: + self.is_running_list.remove(cur_job) + if cur_job in self.waiting_job_list: + self.waiting_job_list.remove(cur_job) + for gpu in cur_job.gpu_ids: + gpu.update_gpu_info(cur_job, 1) + for node in cur_job.node_list: + if cur_job in node.job_run_list: + node.job_run_list.remove(cur_job) + logging.info(f"job {cur_job.clientID} finished") + + + def try_allocate_resource(self, job): + tmp_node_list = self.node_list.copy() + tmp_node_list.sort(key=lambda x: x.free_gpu_num(4)) + tmp_gpu_allocated_list = [] + for node in tmp_node_list: + if node.free_gpu_num(4) >= job.ReqGpuNum:#有可能资源足够，已经是最空闲的节点 + tmp_job_gpu_num = job.ReqGpuNum + tmp_gpu_list = node.gpu_list.copy() + tmp_gpu_list.sort(key=lambda x: x.memory_free, reverse = True) + for gpu in tmp_gpu_list: + try_allocate_flag = gpu.try_to_allocate(job) + if try_allocate_flag == False: + continue + tmp_gpu_allocated_list.append(gpu) + tmp_job_gpu_num -= 1 + if tmp_job_gpu_num == 0: + job.allocate_flag = True + break + if job.allocate_flag == True: + for gpu in tmp_gpu_allocated_list: + gpu.update_gpu_info(job, 0) + job.gpu_ids.append(gpu) + if job in self.waiting_job_list: + self.waiting_job_list.remove(job) + self.is_running_list.append(job) + if job not in self.highpriority_queue: + self.highpriority_queue.append(job) + node.job_run_list.append(job) + job.node_list.append(node) + job.is_running = True + job.ready_replay = True + job.reallocated_flag = True + break + + def try_reallocate_resource(self, job): + tmp_node_list = self.node_list.copy() + tmp_node_list.sort(key=lambda x: x.free_gpu_num(4)) + tmp_gpu_allocated_list = [] + job.gpu_ids.clear() + job.node_list.clear() + for node in tmp_node_list: + if node.free_gpu_num(4) >= job.ReqGpuNum:#有可能资源足够，已经是最空闲的节点 + tmp_job_gpu_num = job.ReqGpuNum + tmp_gpu_list = node.gpu_list.copy() + tmp_gpu_list.sort(key=lambda x: x.memory_free, reverse = True) + for gpu in tmp_gpu_list: + try_allocate_flag = gpu.try_to_allocate(job) + if try_allocate_flag == False: + continue + tmp_gpu_allocated_list.append(gpu) + tmp_job_gpu_num -= 1 + if tmp_job_gpu_num == 0: + job.allocate_flag = True + break + if job.allocate_flag == True: + for gpu in tmp_gpu_allocated_list: + gpu.update_gpu_info(job, 0) + job.gpu_ids.append(gpu) + if job in self.preemt_job_list: + self.preemt_job_list.remove(job) + self.is_running_list.append(job) + node.job_run_list.append(job) + job.node_list.append(node) + job.is_running = True + job.ready_replay = True + job.reallocated_flag = True + job.waiting_round = 0 + break + + def try_preempt_resource(self, job): + logging.info(f"job{job.clientID}try to preempted_resource") + tmp_record = [] + for tmp_job in self.is_running_list: + if tmp_job in self.lowpriority_queue and tmp_job.isDDp == False: + tmp_record.append(tmp_job) + if len(tmp_record) == 0: + return + tmp_record.sort(key=lambda x: x.job_served, reverse = True) + gpu_list = [] + task = {} + m = job.gpu_mem + k = job.ReqGpuNum + for tmp_job_2 in tmp_record: + gpu_id = tmp_job_2.gpu_ids[0].gpu_id + gpu_ip = tmp_job_2.gpu_ids[0].IP_addr + gpu = f"{gpu_ip}:{gpu_id}" + if gpu not in gpu_list: + gpu_list.append(gpu) + id = tmp_job_2.clientID + job_mem = tmp_job_2.gpu_mem + job_served = tmp_job_2.job_served + task[id] = (gpu, job_mem, job_served) + logging.info(f"gpu_list:{gpu_list}") + logging.info(f"task:{task}") + logging.info(f"m:{m} k:{k}") + result = optimize_task_preemption(gpu_list, task, m, k) + job.gpu_ids.clear() + job.node_list.clear() + if result["status"] == "Optimal":#抢占成功 + logging.info(f"job{job.clientID}preempted success......") + tmp_count = len(result["preempted_tasks"]) + self.finish_flag = 0 + for p_job in result["preempted_tasks"]: + pree_job_id = p_job["task_id"] + for i in self.all_job_list: + if i.clientID == pree_job_id: + if i.gpu_ids[0] not in job.gpu_ids: + job.gpu_ids.append(gpu) + gpu = i.gpu_ids[0] + gpu.update_gpu_info(i, 1) + gpu.update_gpu_info(job, 0) + i.gpu_ids.clear() + i.node_list.clear() + i.job_served_time = 0 + self.preemt_job_list.append(i) + if i in self.is_running_list: + self.is_running_list.remove(i) + i.ServerConn.sendall(b"stop") + break + job.allocate_flag = True + while self.finish_flag < tmp_count: + time.sleep(1) + job.is_running = True + job.ready_replay = True + job.reallocated_flag = True + if job in self.waiting_job_list: + self.waiting_job_list.remove(job) + self.is_running_list.append(job) + job.waiting_round = 0 + else: + return + + + def To_Profiling(self, client_id, message): + message = message.split(":", 1)[1].split(',') # 获取 "clientID,gpuCount" + client_id_ = int(message[0].strip()) + req_gpu_num = int(message[1].strip().replace('\x00', '')) + with self.all_job_list_lock: + job = Job(client_id_, req_gpu_num, 0, 0, 0, 0, 0, 0, 0, None, '') + self.all_job_list.append(job) + config = get_config_file() + r = redis_connection() + prof_server_ip = config['ProfilingConfig']['profilingIp_'] + redis_key = f'{prof_server_ip}:{0}'#192.168.0.209:0 + gpu_info_pro = r.hgetall(redis_key) + gpu_info = GPU_info(gpu_info_pro) + tmp_gpu_list = [] + tmp_gpu_list.append(gpu_info) + self.msg_server.redis_conn.publish(self.msg_server.clients[client_id]["status_channel"], "ALLOCATED") + self.msg_server.send_to_client(tmp_gpu_list, client_id) + + + def To_Scheduling(self, client_id, data): + message = data.split(":", 1)[1].split(',') # 获取 "clientID,gpuCount" + client_id_ = int(message[0].strip()) + req_gpu_num = int(message[1].strip().replace('\x00', '')) + model = message[2].strip() + batch_size = int(message[3].strip().replace('\x00', '')) + + result = query_job_info(model, batch_size) + gpu_mem = result.iloc[0]['gpu_mem'] + gpu_util = result.iloc[0]['gpu_util'] + pre_run_time = result.iloc[0]['runtime'] + + with self.all_job_list_lock: + job = Job(client_id_, req_gpu_num, 0, model, 0, batch_size, gpu_mem, gpu_util, pre_run_time, None, '') + if job not in self.all_job_list: + self.all_job_list.append(job) + if job not in self.waiting_job_list: + self.waiting_job_list.append(job) + if job not in self.highpriority_queue: + self.highpriority_queue.append(job) + client_gpu_list = [] + + while job.ready_replay == False: + time.sleep(1) + # self.msg_server.redis_conn.publish(self.msg_server.clients[client_id]["status_channel"], "ALLOCATED") + # if job.allocate_flag == True: + # client_gpu_list = job.gpu_ids.copy() + # else: + # client_gpu_list = [] + client_gpu_list = job.gpu_ids.copy() + # print(client_gpu_list) + self.msg_server.send_to_client(client_gpu_list, client_id_) + if self.logging_level == 1: + self.print_job_all_info(job) + self.print_scheduler_info() + job.ready_replay = False + job.allocate_flag = False + job.reallocated_flag = False + + def Realloc_Resource(self, client_id, message): + data = message.split(":", 1)[1].split(',') + client_id_ = int(data[0].strip()) + req_gpu_num = int(data[1].strip()) + for job in self.all_job_list: + if job.clientID == client_id_: + cur_job = job + break + while cur_job.ready_replay == False: + time.sleep(1) + # self.msg_server.redis_conn.publish(self.msg_server.clients[client_id]["status_channel"], "ALLOCATED") + # if cur_job.reallocated_flag == True: + # client_gpu_list = cur_job.gpu_ids.copy() + # else: + # client_gpu_list = [] + client_gpu_list = job.gpu_ids.copy() + self.msg_server.send_to_client(client_gpu_list, client_id_) + cur_job.ready_replay = False + cur_job.reallocated_flag = False + cur_job.allocate_flag = False + + def print_job_all_info(self, job): + logging.info("====================print job all info====================") + print(job) + if job in self.all_job_list: + logging.info(f"{job.clientID} job in all_job_list") + if job in self.is_running_list: + logging.info(f'{job.clientID} job in is_running_list') + if job in self.highpriority_queue: + logging.info(f'{job.clientID} job in highpriority_queue') + if job in self.lowpriority_queue: + logging.info(f'{job.clientID} job in lowpriority_queue') + if job in self.preemt_job_list: + logging.info(f'{job.clientID} job in preemt_job_list') + if job in self.waiting_job_list: + logging.info(f'{job.clientID} job in waiting_job_list') + for node in job.node_list: + logging.info(f"node:{node.id}") + for gpu in job.gpu_ids: + logging.info(f"gpu:{gpu.IP_addr}:{gpu.gpu_id}") + print("========================================================================") + + def print_scheduler_info(self): + logging.info("====================print scheduler info====================") + for node in self.node_list: + logging.info(f"node:{node.id}:") + for job in node.job_run_list: + logging.info(f"job:{job.clientID}") + for gpu in node.gpu_list: + logging.info(gpu) + for job in gpu.job_list: + logging.info("====================================") + logging.info(job) + logging.info("====================================") + + logging.info("all_job_list:") + for job in self.all_job_list: + logging.info(f"job:{job.clientID}") + logging.info("is_running_list:") + for job in self.is_running_list: + logging.info(f"job:{job.clientID}") + logging.info("highpriority_queue:") + for job in self.highpriority_queue: + logging.info(f"job:{job.clientID}") + logging.info("lowpriority_queue:") + for job in self.lowpriority_queue: + logging.info(f"job:{job.clientID}") + logging.info("preemt_job_list:") + for job in self.preemt_job_list: + logging.info(f"job:{job.clientID}") + logging.info("waiting_job_list:") + for job in self.waiting_job_list: + logging.info(f"job:{job.clientID}") + logging.info("========================================================================") + + def process_message(self, message): + # 处理单条消息的逻辑（线程安全） + channel = message['channel'].decode() + data = message['data'].decode() + if channel == REGISTER_CHANNEL: + self.msg_server.register_client(data) + else: + sender_id = channel.split("_")[-1] + sender_id = int(sender_id) + if data.startswith('TypeA:'): + # self.To_Profiling(sender_id, data) + logging.info(f"Type A :Server processed from {sender_id}: {data}") + self.To_Scheduling(sender_id, data) + elif data.startswith('TypeD:'): + logging.info(f"Server processed from {sender_id}: {data}") + self.Realloc_Resource(sender_id, data) + elif data == 'STOP': + self.clean_client_job(sender_id) + self.msg_server.stop_client(sender_id) + # print(f"Server processed from {sender_id}: {data}") + + def recieve_message_queue(self):#只处理来自proxy的消息 + logging.info("MSG_Server started, listening for registrations and messages") + while self.msg_server.running: + message = self.msg_server.pubsub.get_message(timeout=1.0) + if message and message['type'] == 'message': + self.executor.submit(self.process_message, message) + time.sleep(0.01) + + def Finish_Profiling(self, conn, addr, message): + data = message.split(":", 1)[1].split(',') + client_id = int(data[0].strip()) + req_gpu_num = int(data[1].strip()) + job_runtime = float(data[2].strip()) + gpu_mem = int(data[3].strip()) + gpu_util = int(data[4].strip()) + job_priority = int(data[5].strip()) + for job in self.all_job_list: + if job.clientID == client_id: + job.update_job_info(job_runtime, gpu_mem, gpu_util, job_priority) + break + self.waiting_job_list.append(job) + + def Add_Job_ServerConn(self, conn, addr, message): + data = message.split(":", 1)[1].split(',') + client_id = int(data[0].strip()) + logging.info(f"client_id:{client_id} server connection established") + for job in self.all_job_list: + if job.clientID == client_id and job.ServerConn ==None: + job.ServerConn = conn + break + + def Job_Finish(self, conn, addr, message): + data = message.split(":", 1)[1].split(',') + client_id = int(data[0].strip()) + replay_flag = int(data[1].strip()) + cur_job = None + for job in self.all_job_list: + if job.clientID == client_id: + cur_job = job + break + if cur_job == None: + return + if replay_flag == 1 and cur_job not in self.preemt_job_list:#说明任务没有被抢占，而是正常结束 + with self.is_running_list_lock: + if cur_job in self.is_running_list: + self.is_running_list.remove(cur_job) + with self.highpriority_queue_lock: + if cur_job in self.highpriority_queue: + self.highpriority_queue.remove(cur_job) + with self.lowpriority_queue_lock: + if cur_job in self.lowpriority_queue: + self.lowpriority_queue.remove(cur_job) + with self.all_job_list_lock: + if cur_job in self.all_job_list: + self.all_job_list.remove(cur_job) + with self.waiting_job_list_lock: + if cur_job in self.waiting_job_list: + self.waiting_job_list.remove(cur_job) + with self.preemt_job_list_lock: + if cur_job in self.preemt_job_list: + self.preemt_job_list.remove(cur_job) + for gpu in cur_job.gpu_ids: + gpu.update_gpu_info(cur_job, 1) + for node in cur_job.node_list: + node.job_run_list.remove(cur_job) + r = redis_job_connection() + job_info = { + "job_id": int(cur_job.clientID), + "job_req_gpu_num": int(cur_job.ReqGpuNum), + "job_model": cur_job.model, + "job_batch_size": int(cur_job.batchsize), + "job_gpu_mem": int(cur_job.gpu_mem), + "job_gpu_util": float(cur_job.gpu_util), + "job_runtime": float(cur_job.predict_time), + "job_arrival_time": float(cur_job.arrival_time), + "job_run_time": float(cur_job.run_time), + "job_is_waiting_time": float(cur_job.is_waiting_time), + "job_preempted_time": float(cur_job.preempted_time) + } + job_info_json = json.dumps(job_info) + r.hset(f"{cur_job.clientID}", "job_info", job_info_json) + + + logging.info(f"Job {cur_job.clientID} finished") + logging.info("+++++++++++++++++++++++++++++++++++++++++++++++++++++") + self.print_scheduler_info() + + elif replay_flag == 1 and cur_job in self.preemt_job_list:#说明任务被抢占，需要重新分配资源 + logging.info(f"Job {cur_job.clientID} preempted") + if cur_job in self.is_running_list: + self.is_running_list.remove(cur_job) + if cur_job is not None: + cur_job.ServerConn = None + self.finish_flag += 1 + logging.info(f"finish_flag:{self.finish_flag}") + elif replay_flag == 0: + with self.is_running_list_lock: + if cur_job in self.is_running_list: + self.all_job_list.remove(cur_job) + with self.highpriority_queue_lock: + if cur_job in self.highpriority_queue: + self.highpriority_queue.remove(cur_job) + for gpu in cur_job.gpu_ids: + gpu.update_gpu_info(cur_job, 1) + for node in cur_job.node_list: + node.job_run_list.remove(cur_job) + logging.info(f"job {cur_job.clientID} finished") + self.print_scheduler_info() + + def handle_message(self, conn, addr): + while True: + data = conn.recv(1024) + if not data: + break + message = data.decode() + if message.startswith('TypeB:'): + self.Finish_Profiling(conn, addr, message) + elif message.startswith('TypeC:'): + self.Add_Job_ServerConn(conn, addr, message) + elif message.startswith('TypeE:'): + logging.info("handle_type_E message{}".format(message)) + self.Job_Finish(conn, addr, message) + if self.stop_event.wait(timeout=0.1): + break + conn.close() + logging.info(f"Connection {addr} closed") + + def asyn_scheduler(self): + logging.info("Aysn Scheduler started......") + self.scheduler_running = True + while self.scheduler_running: + logging.info("new round Scheduler running......") + logging.info("------------------------------------------------------------------------") + self.print_gpu_info() + logging.info("------------------------------------------------------------------------") + #1、先检查highpriority_queue中的作业是否已经达到了阈值,是的话先放入到lowpriority_queue中 + logging.info("========================================================================") + logging.info("all job list len:%d",len(self.all_job_list)) + logging.info("highpriority queue len:%d",len(self.highpriority_queue)) + logging.info("lowpriority queue len:%d",len(self.lowpriority_queue)) + logging.info("preemt job list len:%d",len(self.preemt_job_list)) + logging.info("is running list len:%d",len(self.is_running_list)) + logging.info("waiting job list len:%d",len(self.waiting_job_list)) + logging.info("========================================================================") + logging.info("all job list_infomation:") + for it in self.all_job_list: + logging.info(f"job {it.clientID} ReqGpuNum:{it.ReqGpuNum} is_running:{it.is_running} job_served:{it.job_served} is_waiting_time:{it.is_waiting_time} preempted_time:{it.preempted_time}") + logging.info("===========================Running List:===================================") + with self.is_running_list_lock: + for job in self.is_running_list: + job.run_time += self.scheduler_time_interval + job.job_served_time += self.scheduler_time_interval + logging.info(f"job {job.clientID} run time:{job.run_time} reqgpu:{job.ReqGpuNum} job_served:{job.job_served}") + if job in self.highpriority_queue and job.job_served > self.Threshold: + self.highpriority_queue.remove(job) + self.lowpriority_queue.append(job) + #查看被抢占作业是否已经达到了阈值,是的话重新放入到waiting_job_list中 + with self.preemt_job_list_lock: + for job in self.preemt_job_list: + job.preempted_time += self.scheduler_time_interval + job.is_waiting_time += self.scheduler_time_interval + if job.preempted_time > self.Threshold * 0.5: + self.preemt_job_list.remove(job) + self.waiting_job_list.append(job) + if job in self.lowpriority_queue: + self.lowpriority_queue.remove(job) + job.preempted_time = 0 + #2、针对新作业，先检查是否有足够的资源，如果有的话，直接分配资源 + tmp_waiting_queue = self.waiting_job_list.copy() + self.sort_node_list(1, 4, True) + tmp_waiting_queue.sort(key=lambda x: x.arrival_time) + print("waiting queue len:",len(tmp_waiting_queue)) + for job in tmp_waiting_queue: + logging.info("waiting job clientID:%d",job.clientID) + self.try_allocate_resource(job) + # if job.is_running == True: + # self.print_scheduler_info() + #3、检查是否有足够的资源，如果没有的话，检查是否有作业可以抢占资源 + if job.is_running == False and job.waiting_round >= 3: + self.try_preempt_resource(job) + if job.is_running == False: + job.is_waiting_time += self.scheduler_time_interval + job.waiting_round = job.is_waiting_time // 100 + continue + else: + job.is_waiting_time += self.scheduler_time_interval + job.waiting_round = job.is_waiting_time // 100 + # time.sleep(self.scheduler_time_interval) + #4检查被抢占的任务列表是否可以被重新分配资源 + for job in self.preemt_job_list: + logging.info(f"job{job.clientID} in preemt_job_list") + self.try_reallocate_resource(job) + if self.stop_event.wait(timeout=self.scheduler_time_interval): + break + logging.info("aysn Scheduler stopped......") + +def setup_logging_and_output(log_path): + # 清空所有现有的 handlers，避免默认输出到终端 + logging.getLogger().handlers = [] + + # 创建文件 handler + file_handler = logging.FileHandler(log_path) + file_handler.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + file_handler.setFormatter(formatter) + + # 配置 root logger + logger = logging.getLogger() + logger.setLevel(logging.INFO) + logger.addHandler(file_handler) + + # 重定向 sys.stdout 和 sys.stderr 到文件 + log_file = open(log_path, 'a', encoding='utf-8') # 'a' 表示追加模式 + sys.stdout = log_file + sys.stderr = log_file + + # 返回文件对象以便在程序结束时关闭 + return log_file + +def main(): + #输出重定向 + redirect = 0 + if redirect == 1: + # 获取当前文件所在目录 + current_dir = os.path.dirname(os.path.abspath(__file__)) + # 获取上一层目录 + parent_dir = os.path.dirname(current_dir) + # 创建 result_log 文件夹路径 + result_log_dir = os.path.join(parent_dir, "20250306schduleresult_log") + + # 如果文件夹不存在，则创建 + if not os.path.exists(result_log_dir): + os.makedirs(result_log_dir) + + # 生成带有时间戳的文件名并保存到 result_log 文件夹 + current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + log_filename = f"{current_time}_dispatcher.log" + log_path = os.path.join(result_log_dir, log_filename) # 完整日志文件路径 + + # 设置日志和输出重定向 + log_file = setup_logging_and_output(log_path) + + + config = get_config_file() + disp_ip = config['DispatcherConfig']['dpcIp_'] + disp_port = config['DispatcherConfig']['dpcPort_'] + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind((disp_ip, disp_port)) + s.listen() + logging.info("Server started listen at %s:%d", disp_ip, disp_port) + + thread_list = [] + + serverIp = config['ServerConfig']['serverIp_'] + scheduler = Scheduler() + node_1 = Node(0, serverIp) + scheduler.node_list.append(node_1) + for gpu in node_1.gpu_list: + logging.info(f"GPU {gpu.IP_addr}:{gpu.gpu_id} added...") + + + msg_thread = threading.Thread(target=scheduler.recieve_message_queue) + msg_thread.start() + + scheduler_thread = threading.Thread(target=scheduler.asyn_scheduler) + scheduler_thread.start() + + + try: + while True: + conn, addr = s.accept() + logging.info("New connection from %s:%d", addr[0], addr[1]) + t = threading.Thread(target=scheduler.handle_message, args=(conn, addr)) + t.start() + thread_list.append(t) + except KeyboardInterrupt: + scheduler.msg_server.stop() + s.close() + scheduler.stop_event.set() # 设置停止事件 + scheduler.scheduler_running = False + # scheduler_thread.join() # 等待线程结束 + logging.info("Server stopped") + + if redirect == 1: + log_file.close() + # 恢复 sys.stdout 和 sys.stderr（可选，避免影响后续程序） + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/resource_scheduler_dummy.py b/GPU-Virtual-Service/gpu-remoting/scheduler/resource_scheduler_dummy.py new file mode 100644 index 0000000..f90e43f --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/resource_scheduler_dummy.py @@ -0,0 +1,548 @@ +import argparse +import socket +import threading +import json +import redis +from ctypes import Structure, c_char, c_int, c_size_t, sizeof +from cffi import FFI +import struct +import logging +from collections import deque +import time +from multiprocessing import Process +import os +import sys +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime + +# 将 scheduler 目录添加到 sys.path 中 +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.append(parent_dir) + +#自定义类 +from scheduler.util import * +from scheduler.gpu_info import* +from scheduler.job import * +from scheduler.node import * +from scheduler.msg_queue import * + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + + +class Scheduler: + def __init__(self): + # self.cluster_list = [] + # self.cluster_list_lock = threading.Lock() + self.node_list = [] + self.node_list_lock = threading.Lock() + self.all_job_list = [] + self.all_job_list_lock = threading.Lock() + self.allocated_list = [] + self.preemt_job_list = [] + self.preemt_job_list_lock = threading.Lock() + self.is_running_list = [] + self.is_running_list_lock = threading.Lock() + self.waiting_job_list = [] + self.waiting_job_list_lock = threading.Lock() + self.msg_server = MSG_Server() + self.executor = ThreadPoolExecutor(max_workers=5) + self.scheduler_running = False + #scheduler_Queue + self.highpriority_queue = [] + self.highpriority_queue_lock = threading.Lock() + self.lowpriority_queue = [] + self.lowpriority_queue_lock = threading.Lock() + self.Threshold = 600 + self.scheduler_time_interval = 20 + self.stop_event = threading.Event() + self.logging_level = 0 #0:INFO, 1:DEBUG + + + + + + def sort_node_list(self, type, free_gpu_type, rev): + if type == 1:#按照资源最空闲的节点排序 + if rev: + self.node_list.sort(key=lambda x: x.free_gpu_num(free_gpu_type), reverse = True) + else: + self.node_list.sort(key=lambda x: x.free_gpu_num(free_gpu_type)) + elif type == 2:#按照job的数量排序 + if rev: + self.node_list.sort(key=lambda x: x.total_job_num, reverse = True) + else: + self.node_list.sort(key=lambda x: x.total_job_num) + + def has_free_resource(self): + for node in self.node_list: + if node.has_free_gpu(0): + return True + return False + + def try_allocate_resource(self, job): + tmp_node_list = self.node_list.copy() + tmp_node_list.sort(key=lambda x: x.free_gpu_num(4)) + tmp_gpu_allocated_list = [] + for node in tmp_node_list: + if node.free_gpu_num(4) >= job.ReqGpuNum:#有可能资源足够，已经是最空闲的节点 + tmp_job_gpu_num = job.ReqGpuNum + tmp_gpu_list = node.gpu_list.copy() + for gpu in tmp_gpu_list: + try_allocate_flag = gpu.try_to_allocate(job) + if try_allocate_flag == False: + continue + tmp_gpu_allocated_list.append(gpu) + tmp_job_gpu_num -= 1 + if tmp_job_gpu_num == 0: + job.allocate_flag = True + break + if job.allocate_flag == True: + for gpu in tmp_gpu_allocated_list: + gpu.update_gpu_info(job, 0) + job.gpu_ids.append(gpu) + self.waiting_job_list.remove(job) + self.is_running_list.append(job) + self.highpriority_queue.append(job) + node.job_run_list.append(job) + job.node_list.append(node) + job.is_running = True + job.ready_replay = True + break + + def try_preempt_resource(self, job): + tmp_preemted_job_list = [] + tmp_job_gpu_list = [] + + + + def To_Profiling(self, client_id, message): + message = message.split(":", 1)[1].split(',') # 获取 "clientID,gpuCount" + client_id_ = int(message[0].strip()) + req_gpu_num = int(message[1].strip().replace('\x00', '')) + with self.all_job_list_lock: + job = Job(client_id_, req_gpu_num, 0, 0, 0, 0, 0, 0, 0, None, '') + self.all_job_list.append(job) + config = get_config_file() + r = redis_connection() + prof_server_ip = config['ProfilingConfig']['profilingIp_'] + redis_key = f'{prof_server_ip}:{0}'#192.168.0.209:0 + gpu_info_pro = r.hgetall(redis_key) + gpu_info = GPU_info(gpu_info_pro) + tmp_gpu_list = [] + tmp_gpu_list.append(gpu_info) + self.msg_server.redis_conn.publish(self.msg_server.clients[client_id]["status_channel"], "ALLOCATED") + self.msg_server.send_to_client(tmp_gpu_list, client_id) + + + def To_Scheduling(self, client_id, data): + message = data.split(":", 1)[1].split(',') # 获取 "clientID,gpuCount" + client_id_ = int(message[0].strip()) + req_gpu_num = int(message[1].strip().replace('\x00', '')) + model = message[2].strip() + batch_size = int(message[3].strip().replace('\x00', '')) + + result = query_job_info(model, batch_size) + gpu_mem = result.iloc[0]['gpu_mem'] + gpu_util = result.iloc[0]['gpu_util'] + pre_run_time = result.iloc[0]['runtime'] + + with self.all_job_list_lock: + job = Job(client_id_, req_gpu_num, 0, model, 0, batch_size, gpu_mem, gpu_util, pre_run_time, None, '') + if job not in self.all_job_list: + self.all_job_list.append(job) + if job not in self.waiting_job_list: + self.waiting_job_list.append(job) + if job not in self.highpriority_queue: + self.highpriority_queue.append(job) + client_gpu_list = [] + + while job.ready_replay == False: + time.sleep(1) + # self.msg_server.redis_conn.publish(self.msg_server.clients[client_id]["status_channel"], "ALLOCATED") + if job.allocate_flag == True: + client_gpu_list = job.gpu_ids.copy() + else: + client_gpu_list = [] + # print(client_gpu_list) + self.msg_server.send_to_client(client_gpu_list, client_id_) + if self.logging_level == 1: + self.print_job_all_info(job) + self.print_scheduler_info() + job.ready_replay = False + job.allocate_flag = False + + def Realloc_Resource(self, client_id, message): + data = message.split(":", 1)[1].split(',') + client_id_ = int(data[0].strip()) + req_gpu_num = int(data[1].strip()) + for job in self.all_job_list: + if job.clientID == client_id_: + cur_job = job + break + while cur_job.ready_replay == False: + time.sleep(1) + # self.msg_server.redis_conn.publish(self.msg_server.clients[client_id]["status_channel"], "ALLOCATED") + if cur_job.realloacted_flag == True: + client_gpu_list = cur_job.gpu_ids.copy() + else: + client_gpu_list = [] + self.msg_server.send_to_client(client_gpu_list, client_id_) + cur_job.ready_replay = False + cur_job.realloacted_flag = False + + def print_job_all_info(self, job): + logging.info("====================print job all info====================") + print(job) + if job in self.all_job_list: + logging.info(f"{job.clientID} job in all_job_list") + if job in self.highpriority_queue: + logging.info(f'{job.clientID} job in highpriority_queue') + if job in self.lowpriority_queue: + logging.info(f'{job.clientID} job in lowpriority_queue') + if job in self.preemt_job_list: + logging.info(f'{job.clientID} job in preemt_job_list') + if job in self.waiting_job_list: + logging.info(f'{job.clientID} job in waiting_job_list') + for node in job.node_list: + logging.info(f"node:{node.id}") + for gpu in job.gpu_ids: + logging.info(f"gpu:{gpu.IP_addr}:{gpu.gpu_id}") + print("========================================================================") + + def print_scheduler_info(self): + logging.info("====================print scheduler info====================") + for node in self.node_list: + logging.info(f"node:{node.id}:") + for job in node.job_run_list: + logging.info(f"job:{job.clientID}") + for gpu in node.gpu_list: + logging.info(gpu) + for job in gpu.job_list: + logging.info("====================================") + logging.info(job) + logging.info("====================================") + + logging.info("all_job_list:") + for job in self.all_job_list: + logging.info(f"job:{job.clientID}") + logging.info("highpriority_queue:") + for job in self.highpriority_queue: + logging.info(f"job:{job.clientID}") + logging.info("lowpriority_queue:") + for job in self.lowpriority_queue: + logging.info(f"job:{job.clientID}") + logging.info("preemt_job_list:") + for job in self.preemt_job_list: + logging.info(f"job:{job.clientID}") + logging.info("waiting_job_list:") + for job in self.waiting_job_list: + logging.info(f"job:{job.clientID}") + logging.info("========================================================================") + + def process_message(self, message): + # 处理单条消息的逻辑（线程安全） + channel = message['channel'].decode() + data = message['data'].decode() + if channel == REGISTER_CHANNEL: + self.msg_server.register_client(data) + else: + sender_id = channel.split("_")[-1] + if data.startswith('TypeA:'): + # self.To_Profiling(sender_id, data) + logging.info(f"Type A :Server processed from {sender_id}: {data}") + self.To_Scheduling(sender_id, data) + elif data.startswith('TypeD:'): + logging.info(f"Server processed from {sender_id}: {data}") + self.Realloc_Resource(sender_id, data) + elif data == 'STOP': + self.msg_server.stop_client(sender_id) + # print(f"Server processed from {sender_id}: {data}") + + def recieve_message_queue(self):#只处理来自proxy的消息 + logging.info("MSG_Server started, listening for registrations and messages") + while self.msg_server.running: + message = self.msg_server.pubsub.get_message(timeout=1.0) + if message and message['type'] == 'message': + self.executor.submit(self.process_message, message) + time.sleep(0.01) + + def Finish_Profiling(self, conn, addr, message): + data = message.split(":", 1)[1].split(',') + client_id = int(data[0].strip()) + req_gpu_num = int(data[1].strip()) + job_runtime = float(data[2].strip()) + gpu_mem = int(data[3].strip()) + gpu_util = int(data[4].strip()) + job_priority = int(data[5].strip()) + for job in self.all_job_list: + if job.clientID == client_id: + job.update_job_info(job_runtime, gpu_mem, gpu_util, job_priority) + break + self.waiting_job_list.append(job) + + def Add_Job_ServerConn(self, conn, addr, message): + data = message.split(":", 1)[1].split(',') + client_id = int(data[0].strip()) + for job in self.all_job_list: + if job.clientID == client_id and job.ServerConn ==None: + job.ServerConn = conn + break + + def Job_Finish(self, conn, addr, message): + data = message.split(":", 1)[1].split(',') + client_id = int(data[0].strip()) + replay_flag = int(data[1].strip()) + cur_job = None + for job in self.all_job_list: + if job.clientID == client_id: + cur_job = job + break + if cur_job == None: + return + if replay_flag == 1 and cur_job not in self.preemt_job_list:#说明任务没有被抢占，而是正常结束 + with self.is_running_list_lock: + if cur_job in self.is_running_list: + self.all_job_list.remove(cur_job) + with self.highpriority_queue_lock: + if cur_job in self.highpriority_queue: + self.highpriority_queue.remove(cur_job) + with self.lowpriority_queue_lock: + if cur_job in self.lowpriority_queue: + self.lowpriority_queue.remove(cur_job) + with self.all_job_list_lock: + if cur_job in self.all_job_list: + self.all_job_list.remove(cur_job) + for gpu in cur_job.gpu_ids: + gpu.update_gpu_info(cur_job, 1) + for node in cur_job.node_list: + node.job_run_list.remove(cur_job) + logging.info(f"Job {cur_job.clientID} finished") + elif replay_flag == 1 and cur_job in self.preemt_job_list:#说明任务被抢占，需要重新分配资源 + logging.info(f"Job {cur_job.clientID} preempted") + if cur_job is not None: + cur_job.ServerConn = None + elif replay_flag == 0: + with self.is_running_list_lock: + if cur_job in self.is_running_list: + self.all_job_list.remove(cur_job) + with self.highpriority_queue_lock: + if cur_job in self.highpriority_queue: + self.highpriority_queue.remove(cur_job) + for gpu in cur_job.gpu_ids: + gpu.update_gpu_info(cur_job, 1) + for node in cur_job.node_list: + node.job_run_list.remove(cur_job) + logging.info(f"job {cur_job.clientID} finished") + self.print_scheduler_info() + + def handle_message(self, conn, addr): + while True: + data = conn.recv(1024) + if not data: + break + message = data.decode() + if message.startswith('TypeB:'): + self.Finish_Profiling(conn, addr, message) + elif message.startswith('TypeC:'): + self.Add_Job_ServerConn(conn, addr, message) + elif message.startswith('TypeE:'): + logging.info("handle_type_E message") + self.Job_Finish(conn, addr, message) + if self.stop_event.wait(timeout=0.1): + break + conn.close() + logging.info(f"Connection {addr} closed") + + def asyn_scheduler(self): + logging.info("Aysn Scheduler started......") + self.scheduler_running = True + while self.scheduler_running: + logging.info("new round Scheduler running......") + #1、先检查highpriority_queue中的作业是否已经达到了阈值,是的话先放入到lowpriority_queue中 + with self.is_running_list_lock: + for job in self.is_running_list: + job.run_time += self.scheduler_time_interval + if job in self.highpriority_queue and job.job_served > self.Threshold: + self.highpriority_queue.remove(job) + self.lowpriority_queue.append(job) + #查看被抢占作业是否已经达到了阈值,是的话重新放入到waiting_job_list中 + with self.preemt_job_list_lock: + for job in self.preemt_job_list: + job.preempted_time += self.scheduler_time_interval + if job.preempted_time > self.Threshold * 0.5: + self.preemt_job_list.remove(job) + self.waiting_job_list.append(job) + #2、针对新作业，先检查是否有足够的资源，如果有的话，直接分配资源 + tmp_waiting_queue = self.waiting_job_list.copy() + self.sort_node_list(1, 4, True) + tmp_waiting_queue.sort(key=lambda x: x.arrival_time) + print("waiting queue len:",len(tmp_waiting_queue)) + for job in tmp_waiting_queue: + self.try_allocate_resource(job) + #3、检查是否有足够的资源，如果没有的话，检查是否有作业可以抢占资源 + if job.is_running == False and job.waiting_round >= 2:#没有足够的资源,抢占资源 + tmp_running_job_list = self.is_running_list.copy() + tmp_running_job_list.sort(key=lambda x: x.job_served, reverse = True) + get_gpu_num = 0 + while get_gpu_num < job.ReqGpuNum: + for cur_job in tmp_running_job_list: + Msg = f'stop' + cur_job.ServerConn.send(Msg.encode()) + cur_job.realloacted_flag = False + cur_job.ready_replay = False + cur_job.is_running = False + self.is_running_list.remove(cur_job) + self.preemt_job_list.append(cur_job) + for node in job.node_list: + node.job_run_list.remove(cur_job) + for gpu in cur_job.gpu_ids: + gpu.update_gpu_info(cur_job, 1) + allocate_flag_2 = gpu.update_gpu_info(job, 0) + if allocate_flag_2 == False: + continue + job.gpu_ids.append(gpu) + get_gpu_num += 1 + cur_job.gpu_ids.clear() + if get_gpu_num == job.ReqGpuNum: + job.is_running = True + job.realloacted_flag = True + job.ready_replay = True + self.is_running_list.append(job) + self.highpriority_queue.append(job) + logging.info(f'2 Job {job.clientID} is running on node {node.id}') + self.waiting_job_list.remove(job) + node.job_run_list.append(job) + job.node_list.append(node) + elif job.is_running == False and job.waiting_round < 2: + job.waiting_round += 1 + #检查被抢占的作业是否可以重新分配资源 + if self.has_free_resource() and len(self.preemt_job_list) > 0 and len(self.waiting_job_list) == 0: + tmp_preemt_job_list = self.preemt_job_list.copy() + tmp_preemt_job_list.sort(key=lambda x: x.arrival_time) + for preemt_job in tmp_preemt_job_list: + for node in self.node_list: + if node.free_gpu_num(4) >= preemt_job.ReqGpuNum: + tmp_job_gpu_num = preemt_job.ReqGpuNum + tmp_gpu_list = node.gpu_list.copy() + for gpu in tmp_gpu_list: + if tmp_job_gpu_num == 0: + break + + pre_allocate_flag = gpu.update_gpu_info(preemt_job, 0) + if pre_allocate_flag == False: + continue + preemt_job.gpu_ids.append(gpu) + tmp_job_gpu_num -= 1 + preemt_job.is_running = True + preemt_job.realloacted_flag = True + preemt_job.ready_replay = True + logging.info(f'3 Job {preemt_job.clientID} is running on node {node.id}') + self.preemt_job_list.remove(preemt_job) + self.is_running_list.append(preemt_job) + node.job_run_list.append(preemt_job) + preemt_job.node_list.append(node) + break + + # time.sleep(self.scheduler_time_interval) + if self.stop_event.wait(timeout=self.scheduler_time_interval): + break + logging.info("aysn Scheduler stopped......") + +def setup_logging_and_output(log_path): + # 清空所有现有的 handlers，避免默认输出到终端 + logging.getLogger().handlers = [] + + # 创建文件 handler + file_handler = logging.FileHandler(log_path) + file_handler.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + file_handler.setFormatter(formatter) + + # 配置 root logger + logger = logging.getLogger() + logger.setLevel(logging.INFO) + logger.addHandler(file_handler) + + # 重定向 sys.stdout 和 sys.stderr 到文件 + log_file = open(log_path, 'a', encoding='utf-8') # 'a' 表示追加模式 + sys.stdout = log_file + sys.stderr = log_file + + # 返回文件对象以便在程序结束时关闭 + return log_file + +def main(): + #输出重定向 + redirect = 0 + if redirect == 1: + # 获取当前文件所在目录 + current_dir = os.path.dirname(os.path.abspath(__file__)) + # 获取上一层目录 + parent_dir = os.path.dirname(current_dir) + # 创建 result_log 文件夹路径 + result_log_dir = os.path.join(parent_dir, "result_log") + + # 如果文件夹不存在，则创建 + if not os.path.exists(result_log_dir): + os.makedirs(result_log_dir) + + # 生成带有时间戳的文件名并保存到 result_log 文件夹 + current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + log_filename = f"{current_time}_dispatcher.log" + log_path = os.path.join(result_log_dir, log_filename) # 完整日志文件路径 + + # 设置日志和输出重定向 + log_file = setup_logging_and_output(log_path) + + + config = get_config_file() + disp_ip = config['DispatcherConfig']['dpcIp_'] + disp_port = config['DispatcherConfig']['dpcPort_'] + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind((disp_ip, disp_port)) + s.listen() + logging.info("Server started listen at %s:%d", disp_ip, disp_port) + + thread_list = [] + + serverIp = config['ServerConfig']['serverIp_'] + scheduler = Scheduler() + node_1 = Node(0, serverIp) + scheduler.node_list.append(node_1) + for gpu in node_1.gpu_list: + logging.info(f"GPU {gpu.IP_addr}:{gpu.gpu_id} added...") + + + msg_thread = threading.Thread(target=scheduler.recieve_message_queue) + msg_thread.start() + + scheduler_thread = threading.Thread(target=scheduler.asyn_scheduler) + scheduler_thread.start() + + + try: + while True: + conn, addr = s.accept() + logging.info("New connection from %s:%d", addr[0], addr[1]) + t = threading.Thread(target=scheduler.handle_message, args=(conn, addr)) + t.start() + thread_list.append(t) + except KeyboardInterrupt: + scheduler.msg_server.stop() + s.close() + scheduler.stop_event.set() # 设置停止事件 + scheduler.scheduler_running = False + # scheduler_thread.join() # 等待线程结束 + logging.info("Server stopped") + + if redirect == 1: + log_file.close() + # 恢复 sys.stdout 和 sys.stderr（可选，避免影响后续程序） + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/sota_scheduler.py b/GPU-Virtual-Service/gpu-remoting/scheduler/sota_scheduler.py new file mode 100644 index 0000000..3269d3f --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/sota_scheduler.py @@ -0,0 +1,591 @@ +import argparse +import socket +import threading +import json +import redis +from ctypes import Structure, c_char, c_int, c_size_t, sizeof +from cffi import FFI +import struct +import logging +from collections import deque +import time +from multiprocessing import Process +import os +import sys +from concurrent.futures import ThreadPoolExecutor + +# 将 scheduler 目录添加到 sys.path 中 +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.append(parent_dir) + +#自定义类 +from scheduler.util import * +from scheduler.gpu_info import* +from scheduler.job import * +from scheduler.node import * +from scheduler.msg_queue import * + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + + +class Scheduler: + def __init__(self): + # self.cluster_list = [] + # self.cluster_list_lock = threading.Lock() + self.node_list = [] + self.node_list_lock = threading.Lock() + self.all_job_list = [] + self.all_job_list_lock = threading.Lock() + self.allocated_list = [] + self.preemt_job_list = [] + self.preemt_job_list_lock = threading.Lock() + self.is_running_list = [] + self.is_running_list_lock = threading.Lock() + self.waiting_job_list = [] + self.waiting_job_list_lock = threading.Lock() + self.msg_server = MSG_Server() + self.executor = ThreadPoolExecutor(max_workers=5) + self.scheduler_running = False + #scheduler_Queue + self.highpriority_queue = [] + self.highpriority_queue_lock = threading.Lock() + self.lowpriority_queue = [] + self.lowpriority_queue_lock = threading.Lock() + self.Threshold = 600 + self.scheduler_time_interval = 10 + self.stop_event = threading.Event() + self.logging_level = 0 #0:INFO, 1:DEBUG + self.policy = 1#0:FIFO, 1:SJF 2:SRTF 3: + #FIFO + self.fifo_queue = [] + self.fifo_queue_lock = threading.Lock() + #SJF + self.sjf_queue = [] + self.sjf_queue_lock = threading.Lock() + #SRTF + self.srtf_queue = [] + self.srtf_queue_lock = threading.Lock() + #Themis + self.themis_queue = [] + self.themis_queue_lock = threading.Lock() + self.finish_flag = 0 + self.shared_queue = [] + self.exclusive_queue = [] + + + + def try_allocate_resource(self, job): + tmp_node_list = self.node_list.copy() + tmp_node_list.sort(key=lambda x: x.free_gpu_num(4)) + tmp_gpu_allocated_list = [] + for node in tmp_node_list: + if node.free_gpu_num(4) >= job.ReqGpuNum:#有可能资源足够，已经是最空闲的节点 + tmp_job_gpu_num = job.ReqGpuNum + tmp_gpu_list = node.gpu_list.copy() + tmp_gpu_list.sort(key=lambda x: x.memory_free, reverse = True) + for gpu in tmp_gpu_list: + logging.info(f"gpu:{gpu}") + for gpu in tmp_gpu_list: + try_allocate_flag = gpu.try_to_allocate(job) + if try_allocate_flag == False: + continue + tmp_gpu_allocated_list.append(gpu) + tmp_job_gpu_num -= 1 + if tmp_job_gpu_num == 0: + job.allocate_flag = True + break + if job.allocate_flag == True: + for gpu in tmp_gpu_allocated_list: + gpu.update_gpu_info(job, 0) + job.gpu_ids.append(gpu) + if job in self.waiting_job_list: + self.waiting_job_list.remove(job) + if job not in self.is_running_list: + self.is_running_list.append(job) + if job in self.fifo_queue: + self.fifo_queue.remove(job) + if job in self.sjf_queue: + self.sjf_queue.remove(job) + node.job_run_list.append(job) + job.node_list.append(node) + logging.info(f'FIFO Job {job.clientID} is running on node {node.id}') + job.is_running = True + job.ready_replay = True + job.reallocated_flag = True + job.waiting_round = 0 + break + + + def sort_node_list(self, type, free_gpu_type, rev): + if type == 1:#按照资源最空闲的节点排序 + if rev: + self.node_list.sort(key=lambda x: x.free_gpu_num(free_gpu_type), reverse = True) + else: + self.node_list.sort(key=lambda x: x.free_gpu_num(free_gpu_type)) + elif type == 2:#按照job的数量排序 + if rev: + self.node_list.sort(key=lambda x: x.total_job_num, reverse = True) + else: + self.node_list.sort(key=lambda x: x.total_job_num) + + def has_free_resource(self): + for node in self.node_list: + if node.has_free_gpu(4): + return True + return False + + def To_Profiling(self, client_id, message): + message = message.split(":", 1)[1].split(',') # 获取 "clientID,gpuCount" + client_id_ = int(message[0].strip()) + req_gpu_num = int(message[1].strip().replace('\x00', '')) + with self.all_job_list_lock: + job = Job(client_id_, req_gpu_num, 0, 0, 0, 0, 0, 0, 0, None, '') + self.all_job_list.append(job) + config = get_config_file() + r = redis_connection() + prof_server_ip = config['ProfilingConfig']['profilingIp_'] + redis_key = f'{prof_server_ip}:{0}'#192.168.0.209:0 + gpu_info_pro = r.hgetall(redis_key) + gpu_info = GPU_info(gpu_info_pro) + tmp_gpu_list = [] + tmp_gpu_list.append(gpu_info) + self.msg_server.redis_conn.publish(self.msg_server.clients[client_id]["status_channel"], "ALLOCATED") + self.msg_server.send_to_client(tmp_gpu_list, client_id) + + + def To_Scheduling(self, client_id, data): + message = data.split(":", 1)[1].split(',') # 获取 "clientID,gpuCount" + client_id_ = int(message[0].strip()) + req_gpu_num = int(message[1].strip().replace('\x00', '')) + model = message[2].strip() + batch_size = int(message[3].strip().replace('\x00', '')) + + result = query_job_info(model, batch_size) + gpu_mem = result.iloc[0]['gpu_mem'] + gpu_util = result.iloc[0]['gpu_util'] + pre_run_time = result.iloc[0]['runtime'] + + with self.all_job_list_lock: + job = Job(client_id_, req_gpu_num, 0, model, 0, batch_size, gpu_mem, gpu_util, pre_run_time, None, '') + + self.all_job_list.append(job) + self.waiting_job_list.append(job) + self.fifo_queue.append(job) + self.sjf_queue.append(job) + self.srtf_queue.append(job) + self.themis_queue.append(job) + client_gpu_list = [] + while job.ready_replay == False: + time.sleep(1) + # self.msg_server.redis_conn.publish(self.msg_server.clients[client_id]["status_channel"], "ALLOCATED") + # if job.reallocated_flag == True: + # client_gpu_list = job.gpu_ids.copy() + # else: + # client_gpu_list = [] + # print(client_gpu_list) + client_gpu_list = job.gpu_ids.copy() + self.msg_server.send_to_client(client_gpu_list, client_id_) + if self.logging_level == 1: + self.print_job_all_info(job) + self.print_scheduler_info() + job.ready_replay = False + job.reallocated_flag = False + + def Realloc_Resource(self, client_id, message): + data = message.split(":", 1)[1].split(',') + client_id_ = int(data[0].strip()) + req_gpu_num = int(data[1].strip()) + for job in self.all_job_list: + if job.clientID == client_id_: + cur_job = job + break + while cur_job.ready_replay == False: + time.sleep(1) + # self.msg_server.redis_conn.publish(self.msg_server.clients[client_id]["status_channel"], "ALLOCATED") + if cur_job.reallocated_flag == True: + client_gpu_list = cur_job.gpu_ids.copy() + else: + client_gpu_list = [] + self.msg_server.send_to_client(client_gpu_list, client_id_) + cur_job.ready_replay = False + cur_job.reallocated_flag = False + + def print_job_all_info(self, job): + logging.info("====================print job all info====================") + print(job) + if job in self.all_job_list: + logging.info(f"{job.clientID} job in all_job_list") + if job in self.highpriority_queue: + logging.info(f'{job.clientID} job in highpriority_queue') + if job in self.lowpriority_queue: + logging.info(f'{job.clientID} job in lowpriority_queue') + if job in self.preemt_job_list: + logging.info(f'{job.clientID} job in preemt_job_list') + if job in self.waiting_job_list: + logging.info(f'{job.clientID} job in waiting_job_list') + for node in job.node_list: + logging.info(f"node:{node.id}") + for gpu in job.gpu_ids: + logging.info(f"gpu:{gpu.IP_addr}:{gpu.gpu_id}") + print("========================================================================") + + def print_scheduler_info(self): + logging.info("====================print scheduler info====================") + for node in self.node_list: + logging.info(f"node:{node.id}:") + for job in node.job_run_list: + logging.info(f"job:{job.clientID}") + for gpu in node.gpu_list: + logging.info(gpu) + for job in gpu.job_list: + logging.info("====================================") + logging.info(job) + logging.info("====================================") + + logging.info("all_job_list:") + for job in self.all_job_list: + logging.info(f"job:{job.clientID}") + logging.info("highpriority_queue:") + for job in self.highpriority_queue: + logging.info(f"job:{job.clientID}") + logging.info("lowpriority_queue:") + for job in self.lowpriority_queue: + logging.info(f"job:{job.clientID}") + logging.info("preemt_job_list:") + for job in self.preemt_job_list: + logging.info(f"job:{job.clientID}") + logging.info("waiting_job_list:") + for job in self.waiting_job_list: + logging.info(f"job:{job.clientID}") + logging.info("========================================================================") + + def process_message(self, message): + # 处理单条消息的逻辑（线程安全） + channel = message['channel'].decode() + data = message['data'].decode() + if channel == REGISTER_CHANNEL: + self.msg_server.register_client(data) + else: + sender_id = channel.split("_")[-1] + if data.startswith('TypeA:'): + # self.To_Profiling(sender_id, data) + logging.info(f"Server processed from {sender_id}: {data}") + self.To_Scheduling(sender_id, data) + elif data.startswith('TypeD:'): + logging.info(f"Server processed from {sender_id}: {data}") + self.Realloc_Resource(sender_id, data) + # print(f"Server processed from {sender_id}: {data}") + + def recieve_message_queue(self):#只处理来自proxy的消息 + logging.info("MSG_Server started, listening for registrations and messages") + while self.msg_server.running: + message = self.msg_server.pubsub.get_message(timeout=1.0) + if message and message['type'] == 'message': + self.executor.submit(self.process_message, message) + time.sleep(0.01) + + def Finish_Profiling(self, conn, addr, message): + data = message.split(":", 1)[1].split(',') + client_id = int(data[0].strip()) + req_gpu_num = int(data[1].strip()) + job_runtime = float(data[2].strip()) + gpu_mem = int(data[3].strip()) + gpu_util = int(data[4].strip()) + job_priority = int(data[5].strip()) + for job in self.all_job_list: + if job.clientID == client_id: + job.update_job_info(job_runtime, gpu_mem, gpu_util, job_priority) + break + self.waiting_job_list.append(job) + + def Add_Job_ServerConn(self, conn, addr, message): + data = message.split(":", 1)[1].split(',') + client_id = int(data[0].strip()) + logging.info(f"client_id:{client_id} server connection established") + for job in self.all_job_list: + if job.clientID == client_id: + job.ServerConn = conn + break + + def Job_Finish(self, conn, addr, message): + data = message.split(":", 1)[1].split(',') + client_id = int(data[0].strip()) + replay_flag = int(data[1].strip()) + for job in self.all_job_list: + if job.clientID == client_id: + cur_job = job + break + if replay_flag == 1 and cur_job not in self.preemt_job_list:#说明任务没有被抢占，而是正常结束 + with self.is_running_list_lock: + if cur_job in self.is_running_list: + self.is_running_list.remove(cur_job) + with self.fifo_queue_lock: + if cur_job in self.fifo_queue: + self.fifo_queue.remove(cur_job) + with self.sjf_queue_lock: + if cur_job in self.sjf_queue: + self.sjf_queue.remove(cur_job) + with self.all_job_list_lock: + if cur_job in self.all_job_list: + self.all_job_list.remove(cur_job) + for gpu in cur_job.gpu_ids: + gpu.update_gpu_info(cur_job, 1) + for node in cur_job.node_list: + node.job_run_list.remove(cur_job) + + r = redis_job_connection() + job_info = { + "job_id": int(cur_job.clientID), + "job_req_gpu_num": int(cur_job.ReqGpuNum), + "job_model": cur_job.model, + "job_batch_size": int(cur_job.batchsize), + "job_gpu_mem": int(cur_job.gpu_mem), + "job_gpu_util": float(cur_job.gpu_util), + "job_runtime": float(cur_job.predict_time), + "job_arrival_time": float(cur_job.arrival_time), + "job_run_time": float(cur_job.run_time), + "job_is_waiting_time": float(cur_job.is_waiting_time), + "job_preempted_time": float(cur_job.preempted_time) + } + job_info_json = json.dumps(job_info) + r.hset(f"{cur_job.clientID}", "job_info", job_info_json) + + + logging.info(f"Job {cur_job.clientID} finished") + elif replay_flag == 1 and cur_job in self.preemt_job_list:#说明任务被抢占，需要重新分配资源 + logging.info(f"Job {cur_job.clientID} preempted") + elif replay_flag == 0: + with self.is_running_list_lock: + if cur_job in self.is_running_list: + self.all_job_list.remove(cur_job) + with self.highpriority_queue_lock: + if cur_job in self.highpriority_queue: + self.highpriority_queue.remove(cur_job) + for gpu in cur_job.gpu_ids: + gpu.update_gpu_info(cur_job, 1) + for node in cur_job.node_list: + node.job_run_list.remove(cur_job) + logging.info(f"job {cur_job.clientID} finished") + self.print_scheduler_info() + + + + def handle_message(self, conn, addr): + while True: + data = conn.recv(1024) + if not data: + break + message = data.decode() + if message.startswith('TypeB:'): + self.Finish_Profiling(conn, addr, message) + elif message.startswith('TypeC:'): + self.Add_Job_ServerConn(conn, addr, message) + elif message.startswith('TypeE:'): + self.Job_Finish(conn, addr, message) + if self.stop_event.wait(timeout=0.1): + break + conn.close() + logging.info(f"Connection {addr} closed") + + def asyn_scheduler(self): + logging.info("Sota Scheduler started......") + self.scheduler_running = True + while self.scheduler_running: + if self.policy == 0:#FIFO + logging.info("FIFO Scheduler started......") + for i in self.waiting_job_list: + i.is_waiting_time += self.scheduler_time_interval + logging.info(f"job{i.clientID} in waiting_job_list, job waiting time:{i.is_waiting_time}") + for i in self.is_running_list: + i.run_time += self.scheduler_time_interval + logging.info(f"job {i.clientID} run time:{i.run_time} reqgpu:{i.ReqGpuNum}") + + with self.fifo_queue_lock: + tmp_fifo_list = self.fifo_queue.copy() + tmp_fifo_list.sort(key=lambda x: x.arrival_time) + logging.info(f"tmp_fifo_list:{len(tmp_fifo_list)}") + + for job in tmp_fifo_list: + logging.info(f"job:{job.clientID}") + self.try_allocate_resource(job) + + elif self.policy == 1:#SJF + logging.info("SJF Scheduler started......") + for i in self.waiting_job_list: + i.is_waiting_time += self.scheduler_time_interval + logging.info(f"job{i.clientID} in waiting_job_list, job waiting time:{i.is_waiting_time}") + logging.info("=======run job info=======") + for i in self.is_running_list: + i.run_time += self.scheduler_time_interval + logging.info(f"job {i.clientID} run time:{i.run_time} gpu:{i.gpu_ids[0].IP_addr}:{i.gpu_ids[0].gpu_id}") + logging.info("=======gpu info=======") + for gpu in self.node_list[0].gpu_list: + logging.info(f"gpu:{gpu.IP_addr}:{gpu.gpu_id} job_num:{gpu.Job_num}") + for job in gpu.job_list: + logging.info(f"job:{job.clientID}") + + with self.sjf_queue_lock: + tmp_sjf_list = self.sjf_queue.copy() + tmp_sjf_list.sort(key=lambda x: x.predict_time) + logging.info(f"tmp_sjf_list:{len(tmp_sjf_list)}") + + for job in tmp_sjf_list: + logging.info(f"sjf job:{job.clientID}") + self.try_allocate_resource(job) + elif self.policy == 2:#SRTF + logging.info("SRTF Scheduler started......") + with self.waiting_job_list_lock: + tmp_waiting_list = self.waiting_job_list.copy() + tmp_waiting_list.sort(key=lambda x: x.predict_time) + for job in tmp_waiting_list: + if self.has_free_resource(): + for node in self.node_list: + if node.free_gpu_num(0) >= job.ReqGpuNum: + tmp_gpu_num = job.ReqGpuNum + tmp_gpu_list = node.gpu_list.copy() + for gpu in tmp_gpu_list: + if tmp_gpu_num == 0: + break + if gpu.gpu_is_free(3): + gpu.update_gpu_info(job, 0) + job.gpu_ids.append(gpu) + tmp_gpu_num -= 1 + node.job_run_list.append(job) + with self.waiting_job_list_lock: + if job in self.waiting_job_list: + self.waiting_job_list.remove(job) + with self.srtf_queue_lock: + if job in self.srtf_queue: + self.srtf_queue.remove(job) + with self.is_running_list_lock: + self.is_running_list.append(job) + job.is_running = True + job.reallocated_flag = True + job.ready_replay = True + node.job_run_list.append(job) + job.node_list.append(node) + logging.info(f'SRTF Job {job.clientID} is running on node {node.id}') + break + else:#没有空闲资源了，抢占最大的任务 + tmp_running_list = self.is_running_list.copy() + tmp_running_list.sort(key=lambda x: x.predict_time, reverse = True) + preemt_flag = False + while len(tmp_waiting_list) > 0 and not preemt_flag: + for cur_job in tmp_running_list: + if cur_job.predict_time > job.predict_time: + with self.preemt_job_list_lock: + self.preemt_job_list.append(cur_job) + with self.is_running_list_lock: + self.is_running_list.remove(cur_job) + self.is_running_list.append(job) + with self.waiting_job_list_lock: + self.waiting_job_list.remove(job) + self.waiting_job_list.append(cur_job) + for gpu in cur_job.gpu_ids: + gpu.update_gpu_info(cur_job, 1) + gpu.update_gpu_info(job, 0) + job.gpu_ids.append(gpu) + for node in cur_job.node_list: + node.job_run_list.remove(cur_job) + node.job_run_list.append(job) + job.node_list.append(node) + cur_job.is_running = False + cur_job.node_list.clear() + cur_job.gpu_ids.clear() + cur_job.ServerConn.send(b"stop") + job.is_running = True + job.reallocated_flag = True + job.ready_replay = True + logging.info(f'job {job.clientID} preempt {cur_job.clientID}') + break + else: + preemt_flag = True + break + elif self.policy == 3:#Themis + logging.info("Themis Scheduler started......") + self.finish_flag = 0 + tmp_count = 0 + if(len(self.is_running_list)>0): + tmp_running_list = self.is_running_list.copy() + for job in tmp_running_list: + if job.is_exclusive == True: + job.exclusive_time += self.scheduler_time_interval + job.run_time += self.scheduler_time_interval + else : + job.shared_time += self.scheduler_time_interval + job.run_time += self.scheduler_time_interval * 0.5 + if job.run_time > job.predict_time * 0.8: + continue + job.ServerConn.send(b"stop") + tmp_count += 1 + job.is_running = False + job.node_list.clear() + job.gpu_ids.clear() + self.preemt_job_list.append(job) + self.is_running_list.remove(job) + while self.finish_flag < tmp_count: + time.sleep(1) + free_gpu_num = 0 + for node in self.node_list: + free_gpu_num += node.has_free_gpu_num(3) + if free_gpu_num > 1: + exclusive_gpu_num = free_gpu_num // 2 + shared_gpu_num = free_gpu_num - exclusive_gpu_num + tmp_themis_list = self.themis_queue.copy() + tmp_themis_list.sort(key=lambda x: (-x.exclu_shared_ratio, x.arrival_time)) + + + if self.stop_event.wait(timeout=self.scheduler_time_interval): + break + logging.info("aysn Scheduler stopped......") + + + +def main(): + config = get_config_file() + disp_ip = config['DispatcherConfig']['dpcIp_'] + disp_port = config['DispatcherConfig']['dpcPort_'] + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind((disp_ip, disp_port)) + s.listen() + logging.info("Server started listen at %s:%d", disp_ip, disp_port) + + thread_list = [] + + serverIp = config['ServerConfig']['serverIp_'] + scheduler = Scheduler() + node_1 = Node(0, serverIp) + scheduler.node_list.append(node_1) + for gpu in node_1.gpu_list: + logging.info(f"GPU {gpu.IP_addr}:{gpu.gpu_id} added...") + + + msg_thread = threading.Thread(target=scheduler.recieve_message_queue) + msg_thread.start() + + scheduler_thread = threading.Thread(target=scheduler.asyn_scheduler) + scheduler_thread.start() + + + try: + while True: + conn, addr = s.accept() + logging.info("New connection from %s:%d", addr[0], addr[1]) + t = threading.Thread(target=scheduler.handle_message, args=(conn, addr)) + t.start() + thread_list.append(t) + except KeyboardInterrupt: + scheduler.msg_server.stop() + s.close() + scheduler.stop_event.set() # 设置停止事件 + scheduler.scheduler_running = False + scheduler_thread.join() # 等待线程结束 + logging.info("Server stopped") + + + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/GPU-Virtual-Service/gpu-remoting/scheduler/util.py b/GPU-Virtual-Service/gpu-remoting/scheduler/util.py new file mode 100644 index 0000000..8337d2c --- /dev/null +++ b/GPU-Virtual-Service/gpu-remoting/scheduler/util.py @@ -0,0 +1,173 @@ +import argparse +import socket +import threading +import json +import redis +from ctypes import Structure, c_char, c_int, c_size_t, sizeof +from cffi import FFI +import struct +import logging +from collections import deque +import time +from multiprocessing import Process +import os +import sys +import pandas as pd +from scheduler.gpu_info import * +from scheduler.job import * +import pulp + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + +def get_config_file(): + # 获取当前文件的目录 + current_dir = os.path.dirname(os.path.abspath(__file__)) + # 获取上一层目录 + parent_dir = os.path.dirname(current_dir) + # 构建 config.json 文件的路径 + config_path = os.path.join(parent_dir, 'config.json') + with open(config_path, 'r') as f: + config = json.load(f) + + return config + +def query_job_info(model, batch_size): + # 读取 CSV 文件 + df = pd.read_csv('/home/zss/djh_test/NewVersion15/FlexGV_S/scheduler/job_info.csv') + + # 根据 model 和 batch_size 查询 + result = df[(df['model'] == model) & (df['batchsize'] == batch_size)] + + return result + + +def redis_connection(): + config = get_config_file() + # 使用 config 中的配置信息创建 Redis 连接 + r = redis.Redis( + host=config['RedisConfig']['host'], + port=config['RedisConfig']['port'], + db=config['RedisConfig']['db'], + password=config['RedisConfig']['password'] + ) + return r + +def redis_job_connection(): + config = get_config_file() + # 使用 config 中的配置信息创建 Redis 连接 + r = redis.Redis( + host=config['RedisConfig']['host'], + port=config['RedisConfig']['port'], + db=config['RedisConfig']['jobdb'], + password=config['RedisConfig']['password'] + ) + return r + +def allocate_gpus(gpu_free_memory, k, m): + """ + 为新任务分配 GPU 资源。 + :param gpu_free_memory: 字典，键为 GPU ID,值为空闲显存 + :param k: 需要分配的 GPU 数量 + :param m: 每个 GPU 所需的最小显存 + :return: 分配的 GPU 列表，或 None(无解) + """ + # 筛选可用 GPU + available_gpus = {i: free for i, free in gpu_free_memory.items() if free >= m} + + # 检查是否足够 + if len(available_gpus) < k: + print(f"无解：可用 GPU 数量 ({len(available_gpus)}) < 需求 ({k})") + return None + + # 选择 k 个 GPU（此处简单取前 k 个，可按需排序） + allocated_gpus = list(available_gpus.keys())[:k] + + # 输出结果 + print(f"分配的 GPU:") + for gpu in allocated_gpus: + print(f"GPU {gpu}, 空闲显存={gpu_free_memory[gpu]}") + + return allocated_gpus + + + +#多目标整数线性规划 +def optimize_task_preemption(gpus, tasks, m, k): + """ + 优化任务抢占策略：最小化抢占任务数量并最大化服务量。 + + 参数： + gpus (list): GPU ID 列表，例如 ["192.168.0.209:0", "192.168.0.209:1"]。 + tasks (dict): 任务字典，格式 {task_id: (gpu_id, mem, svc)}。 + m (int/float): 每个满足条件的 GPU 所需的最小显存。 + k (int): 需要满足条件的 GPU 数量。 + + 返回： + dict: 包含求解状态、最优任务数量、服务量、抢占任务和满足条件的 GPU。 + """ + prob1 = pulp.LpProblem("Minimize_Task_Count", pulp.LpMinimize) + x1 = {t: pulp.LpVariable(f"x_{t}", cat="Binary") for t in tasks} + y1 = {i: pulp.LpVariable(f"y_{i}", cat="Binary") for i in gpus} + + prob1 += pulp.lpSum(x1[t] for t in tasks) + for i in gpus: + tasks_on_gpu_i = [t for t in tasks if tasks[t][0] == i] + prob1 += pulp.lpSum(tasks[t][1] * x1[t] for t in tasks_on_gpu_i) >= m * y1[i] + prob1 += pulp.lpSum(y1[i] for i in gpus) == k + + prob1.solve() + if pulp.LpStatus[prob1.status] != "Optimal": + return { + "status": pulp.LpStatus[prob1.status], + "message": f"第一阶段无最优解，无法找到最小任务数量的可行解，状态: {pulp.LpStatus[prob1.status]}" + } + + n_min = int(pulp.value(prob1.objective)) + + prob2 = pulp.LpProblem("Maximize_Service_Quantity", pulp.LpMaximize) + x2 = {t: pulp.LpVariable(f"x_{t}", cat="Binary") for t in tasks} + y2 = {i: pulp.LpVariable(f"y_{i}", cat="Binary") for i in gpus} + + prob2 += pulp.lpSum(tasks[t][2] * x2[t] for t in tasks) + for i in gpus: + tasks_on_gpu_i = [t for t in tasks if tasks[t][0] == i] + prob2 += pulp.lpSum(tasks[t][1] * x2[t] for t in tasks_on_gpu_i) >= m * y2[i] + prob2 += pulp.lpSum(y2[i] for i in gpus) == k + prob2 += pulp.lpSum(x2[t] for t in tasks) == n_min + + prob2.solve() + + if pulp.LpStatus[prob2.status] != "Optimal": + return { + "status": pulp.LpStatus[prob2.status], + "message": f"第二阶段无最优解，无法在任务数量 {n_min} 下最大化服务量，状态: {pulp.LpStatus[prob2.status]}" + } + + result = { + "status": "Optimal", + "min_task_count": n_min, + "max_service_quantity": pulp.value(prob2.objective), + "preempted_tasks": [], + "satisfied_gpus": [] + } + + for t in tasks: + if x2[t].value() > 0.5: + result["preempted_tasks"].append({ + "task_id": t, + "gpu_id": tasks[t][0], + "memory": tasks[t][1], + "service": tasks[t][2] + }) + + for i in gpus: + if y2[i].value() > 0.5: + released = sum(tasks[t][1] * x2[t].value() for t in tasks if tasks[t][0] == i) + result["satisfied_gpus"].append({ + "gpu_id": i, + "released_memory": released + }) + + return result +