diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c048201 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +build +build/* +.cache +.clang-format* diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fae95b..640533d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,8 +3,7 @@ cmake_minimum_required(VERSION 3.21) project(CQ-SIMBE VERSION 0.1.0 DESCRIPTION "CQ compliant simulated backend." - LANGUAGES C -) + LANGUAGES C Fortran) # User configure options @@ -41,19 +40,48 @@ option( OFF ) +option( + BUILD_WITH_MPI + "Build with MPI-based communication. Turned ON by default." + OFF +) +message(STATUS "BUILD_WITH_MPI turned ${BUILD_WITH_MPI}.") + +option( + QUEST_BUILT_WITH_MPI + "Flag to define if QuEST was built with MPI. This will affect how we do the communication." + OFF +) +message(STATUS "QUEST_BUILT_WITH_MPI turned ${QUEST_BUILT_WITH_MPI}.") + # Dependencies find_package(Threads REQUIRED) -find_package(QuEST 4.0 REQUIRED) +find_package(QuEST REQUIRED) # CQ-SimBE Library add_library(cq-simbe) add_library(CQ-SIMBE::cq-simbe ALIAS cq-simbe) +if (BUILD_WITH_MPI) + find_package(MPI COMPONENTS Fortran C REQUIRED) + include_directories(SYSTEM ${MPI_INCLUDE_PATH}) + target_compile_definitions(cq-simbe PUBLIC + CQ_WITH_MPI_COMMS=1 + $<$:DEBUG_MODE> + ) +endif() + +if (BUILD_WITH_MPI AND QUEST_BUILT_WITH_MPI) + target_compile_definitions(cq-simbe PRIVATE CQ_CONF_QUEST_WITH_MPI=1) +endif() + + target_compile_features(cq-simbe PRIVATE c_std_99) target_compile_options(cq-simbe PRIVATE -Wall) + target_include_directories(cq-simbe PUBLIC $ @@ -64,6 +92,10 @@ target_include_directories(cq-simbe target_link_libraries(cq-simbe PRIVATE Threads::Threads QuEST::QuEST) +if (BUILD_WITH_MPI) + target_link_libraries(cq-simbe PRIVATE ${MPI_C_LIBRARIES}) +endif() + if (BUILD_FORTRAN_INTERFACE) enable_language(Fortran) diff --git a/examples/fortran/CMakeLists.txt b/examples/fortran/CMakeLists.txt index e8a807a..5feaafe 100644 --- a/examples/fortran/CMakeLists.txt +++ b/examples/fortran/CMakeLists.txt @@ -11,7 +11,11 @@ target_link_libraries(zero-qft ${MATHS_LIBRARY} ) -set_target_properties(zero-qft PROPERTIES Fortran_PREPROCESS ON) +set_target_properties(zero-qft + PROPERTIES + Fortran_PREPROCESS ON + Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/mod/zero-qft" +) add_executable(plus-qft plus_qft.f90 @@ -24,4 +28,28 @@ target_link_libraries(plus-qft ${MATHS_LIBRARY} ) -set_target_properties(plus-qft PROPERTIES Fortran_PREPROCESS ON) +set_target_properties(plus-qft + PROPERTIES + Fortran_PREPROCESS ON + Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/mod/plus-qft" +) + +if (BUILD_WITH_MPI) + add_executable(mpi-plus-qft + mpi_plus_qft.f90 + utils.f90 + ) + + target_link_libraries(mpi-plus-qft + PRIVATE + CQ-SIMBE::cq-simbe + ${MATHS_LIBRARY} + ${MPI_Fortran_LIBRARIES} + ) + + set_target_properties(mpi-plus-qft + PROPERTIES + Fortran_PREPROCESS ON + Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/mod/mpi-plus-qft" + ) +endif() diff --git a/examples/fortran/mpi_plus_qft.f90 b/examples/fortran/mpi_plus_qft.f90 new file mode 100644 index 0000000..761f2dc --- /dev/null +++ b/examples/fortran/mpi_plus_qft.f90 @@ -0,0 +1,129 @@ +program plus_qft +use mpi +use cq +#include "cqf.h" +use example_utils +implicit none + +integer :: ireturn, freturn, alloc_status, free_status, reg_status, qrun_status, ierr +integer(kind=8) :: NQUBITS, NSHOTS, NMEASURE +type(qubit) :: qrc +integer(kind=2), allocatable, target :: cr(:) + +integer :: id, nproc, split_cond, cq_comm + +call MPI_Init(ierr) +call MPI_Comm_size(MPI_COMM_WORLD, nproc, ierr) +call MPI_Comm_rank(MPI_COMM_WORLD, id, ierr) + +if (id >= 3) then + split_cond = 1 +else + split_cond = 0 +end if + +call MPI_Comm_split(MPI_COMM_WORLD, split_cond, id, cq_comm, ierr) + +NQUBITS = 10 +NSHOTS = 10 +NMEASURE = NQUBITS + +write(*,'(A)') 'before init' + +if (split_cond == 1) then +ireturn = cq_init_custom_mpi_comm(cq_comm, 1) + +write(*,'(A,I4)') 'cq_init returned: ',ireturn + +reg_status = cq_register_qkern(plus_state_qft) + +write(*,'(A,I4)') 'after register_qkern: ', reg_status + +CQ_PROG_BEGIN() + +alloc_status = cq_alloc_qureg(qrc, NQUBITS) + +write(*,'(A,I4)') 'alloc_qureg returned: ', alloc_status + +allocate(cr(NMEASURE * NSHOTS)) + +CALL cq_init_creg(NMEASURE * NSHOTS, -1, cr) + +write(*,'(A)') 'after init_creg' + +qrun_status = cq_sm_qrun(plus_state_qft, qrc, NQUBITS, cr, NMEASURE, NSHOTS) + +write(*,'(A,I4)') 'after sm_qrun: ', qrun_status + +CALL report_results(cr, NMEASURE, NSHOTS) + +write(*,'(A)') 'after report results' + +free_status = cq_free_qureg(qrc) + +write(*,'(A,I4)') 'free_status returned: ', free_status + +CQ_PROG_END() + +freturn = cq_finalise(0) + +write(*,'(A,I4)') 'cq_finalise returned: ',freturn + +write(*,'(A)') 'after finalise' + +end if + +if (split_cond == 0) then + write(*,*) 'I am user-owned rank doing something unrelated to CQ' +end if + +call MPI_Barrier(MPI_COMM_WORLD, ierr) +call MPI_Finalize(ierr) + +contains + + subroutine qft(NQUBITS, qr) + implicit none + integer(kind=8), value :: NQUBITS + type(qubit), value :: qr + integer(kind=8) :: i, j + integer :: status + real(8), parameter :: PI = 3.1415926535897932384626433832795028841971694 + real(8) :: angle + + do i = 0, NQUBITS - 1 + status = cq_hadamard(qr, i) + do j = i + 1, NQUBITS - 1 + angle = PI / (2.0 ** j) + status = cq_cphase(qr, j, i, angle) + end do + end do + + do i = 0, (NQUBITS / 2) - 1 + j = NQUBITS - (i + 1) + status = cq_swap(qr, i, j) + end do + end subroutine qft + + function plus_state_qft(NQUBITS, qr, NMEASURE, cr, reg) bind(C) result(status) + implicit none + integer(kind=8), value :: NQUBITS + type(qubit), value :: qr + integer(kind=8), value :: NMEASURE + integer(kind=2), intent(inout) :: cr(0:NMEASURE) + type(qkern_map), value :: reg + integer(kind=8) :: i + integer :: status + integer(kind=8) :: STATE_IDX = 0 + CQ_REGISTER_KERNEL("plus_state_qft", reg) + status = cq_set_qureg(qr, STATE_IDX, NQUBITS) + + do i = 0, NQUBITS-1 + status = cq_hadamard(qr, i) + end do + + call qft(NQUBITS, qr) + status = cq_measure_qureg(qr, NQUBITS, cr) + end function plus_state_qft + +end program diff --git a/examples/fortran/plus_qft.f90 b/examples/fortran/plus_qft.f90 index 222e7df..7c92fcb 100644 --- a/examples/fortran/plus_qft.f90 +++ b/examples/fortran/plus_qft.f90 @@ -19,6 +19,12 @@ program plus_qft write(*,'(A,I4)') 'cq_init returned: ',ireturn +reg_status = cq_register_qkern(plus_state_qft) + +write(*,'(A,I4)') 'after register_qkern: ', reg_status + +CQ_PROG_BEGIN() + alloc_status = cq_alloc_qureg(qrc, NQUBITS) write(*,'(A,I4)') 'alloc_qureg returned: ', alloc_status @@ -29,10 +35,6 @@ program plus_qft write(*,'(A)') 'after init_creg' -reg_status = cq_register_qkern(plus_state_qft) - -write(*,'(A,I4)') 'after register_qkern: ', reg_status - qrun_status = cq_sm_qrun(plus_state_qft, qrc, NQUBITS, cr, NMEASURE, NSHOTS) write(*,'(A,I4)') 'after sm_qrun: ', qrun_status @@ -45,6 +47,8 @@ program plus_qft write(*,'(A,I4)') 'free_status returned: ', free_status +CQ_PROG_END() + freturn = cq_finalise(0) write(*,'(A,I4)') 'cq_finalise returned: ',freturn diff --git a/examples/fortran/zero_qft.f90 b/examples/fortran/zero_qft.f90 index 6e5558a..b146ee1 100644 --- a/examples/fortran/zero_qft.f90 +++ b/examples/fortran/zero_qft.f90 @@ -15,10 +15,16 @@ program zero_qft write(*,'(A)') 'before init' -ireturn = cq_init(0) +ireturn = cq_init(1) write(*,'(A,I4)') 'cq_init returned: ',ireturn +reg_status = cq_register_qkern(zero_state_qft) + +write(*,'(A,I4)') 'after register_qkern: ', reg_status + +CQ_PROG_BEGIN() + alloc_status = cq_alloc_qureg(qrc, NQUBITS) write(*,'(A,I4)') 'alloc_qureg returned: ', alloc_status @@ -29,11 +35,6 @@ program zero_qft write(*,'(A)') 'after init_creg' - -reg_status = cq_register_qkern(zero_state_qft) - -write(*,'(A,I4)') 'after register_qkern: ', reg_status - qrun_status = cq_sm_qrun(zero_state_qft, qrc, NQUBITS, cr, NMEASURE, NSHOTS) write(*,'(A,I4)') 'after sm_qrun: ', qrun_status @@ -46,7 +47,9 @@ program zero_qft write(*,'(A,I4)') 'free_status returned: ', free_status -freturn = cq_finalise(0) +CQ_PROG_END() + +freturn = cq_finalise(1) write(*,'(A,I4)') 'cq_finalise returned: ',freturn diff --git a/examples/qft/CMakeLists.txt b/examples/qft/CMakeLists.txt index 64150b6..ff07a54 100644 --- a/examples/qft/CMakeLists.txt +++ b/examples/qft/CMakeLists.txt @@ -22,4 +22,20 @@ target_link_libraries(aqft PRIVATE CQ-SIMBE::cq-simbe ${MATHS_LIBRARY} -) \ No newline at end of file +) + +if (BUILD_WITH_MPI) + add_executable(mpi_aqft + mpi_aqft.c + qft-utils.c + ) + + find_library(MATHS_LIBRARY m REQUIRED) + + target_link_libraries(mpi_aqft + PRIVATE + CQ-SIMBE::cq-simbe + ${MATHS_LIBRARY} + ${MPI_C_LIBRARIES} + ) +endif() diff --git a/examples/qft/aqft.c b/examples/qft/aqft.c index 863ab6f..732006d 100644 --- a/examples/qft/aqft.c +++ b/examples/qft/aqft.c @@ -1,11 +1,12 @@ -#include -#include -#include #include "qft.h" + #include "cq.h" -int main (void) -{ +#include +#include +#include + +int main(void) { const size_t NQUBITS = 10; const size_t NSHOTS = 10; const size_t NMEASURE = NQUBITS; @@ -14,6 +15,11 @@ int main (void) cq_init(0); + register_qkern(zero_init_full_qft); + register_qkern(plus_init_full_qft); + + CQ_PROG_BEGIN() + // We will reuse the quantum buffer as the quantum // kernels cannot run simultaneously (for now...) qubit * qr = NULL; @@ -25,29 +31,28 @@ int main (void) init_creg(NMEASURE * NSHOTS, -1, cr_zero); init_creg(NMEASURE * NSHOTS, -1, cr_plus); - register_qkern(zero_init_full_qft); - register_qkern(plus_init_full_qft); - printf("Offloading both QFT circuits to the quantum device.\n"); am_qrun(zero_init_full_qft, qr, NQUBITS, cr_zero, NMEASURE, NSHOTS, &eh_zero); am_qrun(plus_init_full_qft, qr, NQUBITS, cr_plus, NMEASURE, NSHOTS, &eh_plus); printf("Hello from the host, pretend I'm doing something useful!\n"); sleep(2); - printf("Hello again, I'm done being 'useful' and will now wait for "); + printf("Hello again, I'm done being 'useful' and will now wait for "); printf("the quantum device to return!\n"); wait_qrun(&eh_zero); printf("Results from zero-initialised QFT:\n"); report_results(cr_zero, NMEASURE, NSHOTS); - wait_qrun(&eh_plus); + wait_qrun(&eh_plus); printf("Results from plus-initialised QFT:\n"); report_results(cr_plus, NMEASURE, NSHOTS); free_qureg(&qr); + CQ_PROG_END() + cq_finalise(0); return 0; -} \ No newline at end of file +} diff --git a/examples/qft/mpi_aqft.c b/examples/qft/mpi_aqft.c new file mode 100644 index 0000000..bc520c1 --- /dev/null +++ b/examples/qft/mpi_aqft.c @@ -0,0 +1,86 @@ +#include "env.h" +#include "qft.h" + +#include "cq.h" + +#include +#include +#include + +#include + +int main(void) { + int provided; + + MPI_Comm cq_comm; + int nproc; + int rank; + MPI_Init_thread(NULL, NULL, MPI_THREAD_MULTIPLE, &provided); + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + const int split_cond = rank >= 3; + MPI_Comm_split(MPI_COMM_WORLD, split_cond, rank, &cq_comm); + const size_t NQUBITS = 10; + const size_t NSHOTS = 10; + const size_t NMEASURE = NQUBITS; + + cstate cr_zero[NMEASURE * NSHOTS]; + cstate cr_plus[NMEASURE * NSHOTS]; + cq_exec eh_zero, eh_plus; + + qubit * qr = NULL; + + if (split_cond) { + cq_init_custom_mpi_comm(cq_comm, 1); + + register_qkern(zero_init_full_qft); + register_qkern(plus_init_full_qft); + + CQ_PROG_BEGIN() + + // We will reuse the quantum buffer as the quantum + // kernels cannot run simultaneously (for now...) + alloc_qureg(&qr, NQUBITS); + + init_creg(NMEASURE * NSHOTS, -1, cr_zero); + init_creg(NMEASURE * NSHOTS, -1, cr_plus); + + printf("Offloading both QFT circuits to the quantum device.\n"); + am_qrun(zero_init_full_qft, qr, NQUBITS, cr_zero, NMEASURE, NSHOTS, + &eh_zero); + am_qrun(plus_init_full_qft, qr, NQUBITS, cr_plus, NMEASURE, NSHOTS, + &eh_plus); + + CQ_PROG_END() + } + + if (rank <= 3) { + printf("Hello from the host, pretend I'm doing something useful!\n"); + sleep(2); + printf("Hello again, I'm done being 'useful' and will now wait for "); + printf("the quantum device to return!\n"); + } + + if (split_cond) { + CQ_PROG_BEGIN() + + wait_qrun(&eh_zero); + wait_qrun(&eh_plus); + + free_qureg(&qr); + + printf("Results from zero-initialised QFT:\n"); + report_results(cr_zero, NMEASURE, NSHOTS); + + printf("Results from plus-initialised QFT:\n"); + report_results(cr_plus, NMEASURE, NSHOTS); + + CQ_PROG_END() + cq_finalise(0); + } + + MPI_Finalize(); + + return 0; +} diff --git a/examples/qft/sqft.c b/examples/qft/sqft.c index c5db0bd..7e49a2b 100644 --- a/examples/qft/sqft.c +++ b/examples/qft/sqft.c @@ -1,16 +1,19 @@ -#include #include -#include "qft.h" +#include #include "cq.h" +#include "qft.h" -int main (void) -{ +int main(void) { const size_t NQUBITS = 10; const size_t NSHOTS = 10; const size_t NMEASURE = NQUBITS; cq_init(0); + register_qkern(zero_init_full_qft); + register_qkern(plus_init_full_qft); + CQ_PROG_BEGIN(); + qubit * qr = NULL; alloc_qureg(&qr, NQUBITS); @@ -18,9 +21,6 @@ int main (void) cr = malloc(NMEASURE * NSHOTS * sizeof(cstate)); init_creg(NMEASURE * NSHOTS, -1, cr); - register_qkern(zero_init_full_qft); - register_qkern(plus_init_full_qft); - printf("Running first QFT circuit on quantum device.\n"); sm_qrun(zero_init_full_qft, qr, NQUBITS, cr, NMEASURE, NSHOTS); report_results(cr, NMEASURE, NSHOTS); @@ -32,7 +32,9 @@ int main (void) free_qureg(&qr); free(cr); + CQ_PROG_END(); + cq_finalise(0); return 0; -} \ No newline at end of file +} diff --git a/examples/vqe/vqe.c b/examples/vqe/vqe.c index 778dc06..ba55c06 100644 --- a/examples/vqe/vqe.c +++ b/examples/vqe/vqe.c @@ -1,32 +1,26 @@ -#include "cq.h" #include "vqe.h" +#include "cq.h" #include #include -hamiltonian h2_hamil = { - .paulis = { - 'I', 'I', - 'I', 'Z', - 'Z', 'I', - 'Z', 'Z', - 'X', 'X' }, - .coeffs = { - -1.052373245772859, - 0.39793742484318045, - -0.39793742484318045, - -0.01128010425623538, - 0.18093119978423156}, - .term_start_idx = 0 -}; - -int main (void) { +hamiltonian h2_hamil + = { .paulis = { 'I', 'I', 'I', 'Z', 'Z', 'I', 'Z', 'Z', 'X', 'X' }, + .coeffs + = { -1.052373245772859, 0.39793742484318045, -0.39793742484318045, + -0.01128010425623538, 0.18093119978423156 }, + .term_start_idx = 0 }; + +int main(void) { const double TRUE_ENERGY = -1.85728; const size_t NQUBITS = 2; const size_t NSHOTS = 2000; const size_t NMEASURE = NQUBITS; cq_init(0); + register_pqkern(ansatz); + + CQ_PROG_BEGIN(); qubit * qr = NULL; alloc_qureg(&qr, NQUBITS); @@ -35,19 +29,18 @@ int main (void) { cr = malloc(NMEASURE * NSHOTS * sizeof(cstate)); init_creg(NMEASURE * NSHOTS, -1, cr); - register_qkern(ansatz); - double energy = vqe_optimize(qr, cr, NQUBITS, NMEASURE, NSHOTS); - printf("FCI Energy: %f\n" - "VQE Energy: %f\n" - "VQE Error: %f\n", - TRUE_ENERGY, - energy, - TRUE_ENERGY - energy); - + printf( + "FCI Energy: %f\n" + "VQE Energy: %f\n" + "VQE Error: %f\n", + TRUE_ENERGY, energy, TRUE_ENERGY - energy); + free_qureg(&qr); free(cr); + CQ_PROG_END(); + cq_finalise(0); return 0; diff --git a/examples/vqe/vqe.h b/examples/vqe/vqe.h index 5be006c..2bbdf83 100644 --- a/examples/vqe/vqe.h +++ b/examples/vqe/vqe.h @@ -9,25 +9,22 @@ #define M_PI 3.141592653589793238462643383 #endif -typedef struct vqe_settings { - qubit * qr; - cstate * cr; - size_t NQUBITS; - size_t NSHOTS; -} vqe_settings; - #define NLAYERS 1 #define MAX_NQUBITS 2 #define NPARAMS NLAYERS * MAX_NQUBITS #define PARAM_MIN 0.0 #define PARAM_MAX 2.0 * M_PI -typedef struct vqe_context { +typedef struct vqe_settings { + qubit * qr; + cstate * cr; + size_t NQUBITS; + size_t NSHOTS; double prev_energy; double params[NPARAMS]; double prev_params[NPARAMS]; ptrdiff_t iter; -} vqe_context; +} vqe_settings; #define NTERMS 5 #define NPAULIS MAX_NQUBITS * NTERMS @@ -40,18 +37,43 @@ typedef struct hamiltonian { extern hamiltonian h2_hamil; void init_hf_state(qubit * qr, int num_spin_orbitals); -cq_status ansatz(const size_t NQUBITS, qubit * qr, const size_t NMEASURE, cstate * cr, qkern_map * reg); +cq_status ansatz(const size_t NQUBITS, + qubit * qr, + const size_t NMEASURE, + cstate * cr, + void * kernpar, + pqkern_map * reg); + +static double get_term_expectation(int * histogram, + int num_bins, + const size_t NMEASURE, + const size_t NSHOTS, + double coeff); + +void init_vqe_settings(vqe_settings * settings, + qubit * qr, + cstate * cr, + const size_t NQUBITS, + const size_t NSHOTS); + +void init_vqe_params(double * params, ptrdiff_t num_params); + +double vqe_iter(qubit * qr, + cstate * cr, + const size_t NQUBITS, + const size_t NSHOTS, + const double * x); -static double get_term_expectation(int *histogram, int num_bins, - const size_t NMEASURE, const size_t NSHOTS, double coeff); +double vqe_iter_nlopt(unsigned int n, + const double * x, + double * grad, + void * f_data); -void init_vqe_settings(vqe_settings *settings, qubit * qr, cstate * cr, - const size_t NQUBITS, const size_t NSHOTS); -void init_vqe_params(double *params, ptrdiff_t num_params); -double vqe_iter(qubit * qr, cstate * cr, const size_t NQUBITS, const size_t NSHOTS); -double vqe_iter_nlopt(unsigned int n, const double *x, double *grad, void *f_data); -double vqe_optimize(qubit * qr, cstate * cr, const size_t NQUBITS, - const size_t NMEASURE, const size_t NSHOTS); +double vqe_optimize(qubit * qr, + cstate * cr, + const size_t NQUBITS, + const size_t NMEASURE, + const size_t NSHOTS); #endif diff --git a/examples/vqe/vqe_utils.c b/examples/vqe/vqe_utils.c index 8682c38..47f2e82 100644 --- a/examples/vqe/vqe_utils.c +++ b/examples/vqe/vqe_utils.c @@ -1,19 +1,24 @@ -#include "vqe.h" #include +#include "vqe.h" #include "nlopt.h" -vqe_context context = {0}; - void init_hf_state(qubit * qr, int num_spin_orbitals) { for (ptrdiff_t i = 0; i < num_spin_orbitals; ++i) { paulix(&qr[i]); } } -cq_status ansatz(const size_t NQUBITS, qubit * qr, const size_t NMEASURE, cstate * cr, qkern_map * reg) { +cq_status ansatz(const size_t NQUBITS, + qubit * qr, + const size_t NMEASURE, + cstate * cr, + void * kernpar, + pqkern_map * reg) { CQ_REGISTER_KERNEL(reg) + double * params = (double *)kernpar; + HANDLE_CQ_ERROR(set_qureg(qr, 0, NQUBITS)); // HF init state. For H2 we set NQUBITS/2 to 1 init_hf_state(qr, NQUBITS / 2); @@ -22,28 +27,28 @@ cq_status ansatz(const size_t NQUBITS, qubit * qr, const size_t NMEASURE, cstate for (ptrdiff_t i = 0; i < NLAYERS; ++i) { for (ptrdiff_t j = 0; j < NQUBITS; ++j) { ptrdiff_t param_idx = j + i * (NQUBITS * 2); - roty(&qr[j], context.params[param_idx]); + roty(&qr[j], params[param_idx]); if (j < NQUBITS - 1) { int control = j; - int target = j + 1; - cpauliz(&qr[control], &qr[target]); + int target = j + 1; + cpauliz(&qr[control], &qr[target]); } // Changing basis to get expectation value if (i == NLAYERS - 1) { - char pauli = h2_hamil.paulis[h2_hamil.term_start_idx + j]; + char pauli = h2_hamil.paulis[h2_hamil.term_start_idx + j]; if (pauli == 'X') { hadamard(&qr[j]); measure_qubit(&qr[j], &cr[j]); - + } else if (pauli == 'Y') { rotx(&qr[j], M_PI / 2.0); measure_qubit(&qr[j], &cr[j]); - + } else if (pauli == 'Z') { measure_qubit(&qr[j], &cr[j]); - + } else { // On I -- do nothing } @@ -62,8 +67,11 @@ static int count_ones(int dec_idx, const size_t NMEASURE) { return num_ones; } -double get_term_expectation(int *histogram, int num_bins, - const size_t NMEASURE, const size_t NSHOTS, double coeff) { +double get_term_expectation(int * histogram, + int num_bins, + const size_t NMEASURE, + const size_t NSHOTS, + double coeff) { double term_expectation = 0.0; for (ptrdiff_t i = 0; i < num_bins; ++i) { @@ -79,7 +87,7 @@ double get_term_expectation(int *histogram, int num_bins, } #define EXAMPLE_SEED 102141 -void init_vqe_params(double *params, ptrdiff_t num_params) { +void init_vqe_params(double * params, ptrdiff_t num_params) { const double min_val = PARAM_MIN; const double max_val = PARAM_MAX; const int min = 0; @@ -94,15 +102,18 @@ void init_vqe_params(double *params, ptrdiff_t num_params) { } #undef EXAMPLE_SEED -void init_vqe_settings(vqe_settings *settings, qubit * qr, cstate * cr, - const size_t NQUBITS, const size_t NSHOTS) { +void init_vqe_settings(vqe_settings * settings, + qubit * qr, + cstate * cr, + const size_t NQUBITS, + const size_t NSHOTS) { settings->qr = qr; settings->cr = cr; settings->NQUBITS = NQUBITS; settings->NSHOTS = NSHOTS; } -static int result_to_int(cstate *cr, int num_qubits) { +static int result_to_int(cstate * cr, int num_qubits) { int result = 0; int base = 1; for (ptrdiff_t i = num_qubits - 1; i >= 0; --i) { @@ -116,8 +127,12 @@ static int result_to_int(cstate *cr, int num_qubits) { #define MAX_BINS 1 << MAX_NQUBITS -double vqe_iter(qubit * qr, cstate * cr, const size_t NQUBITS, const size_t NSHOTS) { - int histogram[MAX_BINS] = {0}; +double vqe_iter(qubit * qr, + cstate * cr, + const size_t NQUBITS, + const size_t NSHOTS, + const double * x) { + int histogram[MAX_BINS] = { 0 }; const ptrdiff_t num_bins = (ptrdiff_t)1 << NQUBITS; double expectation = 0.0; const size_t NMEASURE = NQUBITS; @@ -126,7 +141,8 @@ double vqe_iter(qubit * qr, cstate * cr, const size_t NQUBITS, const size_t NSHO for (ptrdiff_t i = 0; i < NTERMS; ++i) { ptrdiff_t paulis_start = i * NQUBITS; - sm_qrun(ansatz, qr, NQUBITS, cr, NMEASURE, NSHOTS); + smp_qrun(ansatz, x, NPARAMS * sizeof(double), qr, NQUBITS, cr, NMEASURE, + NSHOTS); for (ptrdiff_t j = 0; j < NSHOTS; ++j) { ptrdiff_t cr_start = j * NQUBITS; @@ -134,56 +150,69 @@ double vqe_iter(qubit * qr, cstate * cr, const size_t NQUBITS, const size_t NSHO } double coeff = h2_hamil.coeffs[i]; - expectation += get_term_expectation(histogram, num_bins, NMEASURE, NSHOTS, coeff); + expectation + += get_term_expectation(histogram, num_bins, NMEASURE, NSHOTS, coeff); h2_hamil.term_start_idx += NQUBITS; } - printf("Iter: %td -- Expectation: %f\n", context.iter, expectation); - ++context.iter; return expectation; } #undef MAX_BINS -double vqe_iter_nlopt(unsigned int n, const double *x, double *grad, void *f_data) { - vqe_settings *settings = (vqe_settings *)(f_data); +double vqe_iter_nlopt(unsigned int n, + const double * x, + double * grad, + void * f_data) { + vqe_settings * settings = (vqe_settings *)(f_data); + + double energy = vqe_iter(settings->qr, settings->cr, settings->NQUBITS, + settings->NSHOTS, x); + + printf("Iter: %td -- Expectation: %f\n", settings->iter, energy); + ++settings->iter; - double energy = vqe_iter(settings->qr, settings->cr, settings->NQUBITS, settings->NSHOTS); if (!grad) return energy; // estimate gradient if using gradient-based optimizer for (ptrdiff_t i = 0; i < NPARAMS; ++i) { - grad[i] = (energy - context.prev_energy) / (x[i] - context.prev_params[i]); - context.prev_params[i] = x[i]; + grad[i] + = (energy - settings->prev_energy) / (x[i] - settings->prev_params[i]); + settings->prev_params[i] = x[i]; } - context.prev_energy = energy; + settings->prev_energy = energy; return energy; } -double vqe_optimize(qubit * qr, cstate * cr, const size_t NQUBITS, - const size_t NMEASURE, const size_t NSHOTS) { - init_vqe_params(context.params, NPARAMS); +double vqe_optimize(qubit * qr, + cstate * cr, + const size_t NQUBITS, + const size_t NMEASURE, + const size_t NSHOTS) { + vqe_settings settings = { 0 }; + init_vqe_settings(&settings, qr, cr, NQUBITS, NSHOTS); + + init_vqe_params(settings.params, NPARAMS); + // gradient-free optimizers nlopt_opt opt = nlopt_create(NLOPT_LN_COBYLA, NPARAMS); - //nlopt_opt opt = nlopt_create(NLOPT_LN_SBPLX, NPARAMS); - + // nlopt_opt opt = nlopt_create(NLOPT_LN_SBPLX, NPARAMS); + // gradient-based optimizers - //nlopt_opt opt = nlopt_create(NLOPT_LD_LBFGS, NPARAMS); - //nlopt_opt opt = nlopt_create(NLOPT_LD_SLSQP, NPARAMS); - + // nlopt_opt opt = nlopt_create(NLOPT_LD_LBFGS, NPARAMS); + // nlopt_opt opt = nlopt_create(NLOPT_LD_SLSQP, NPARAMS); + nlopt_set_maxeval(opt, 200); double ftol = 0.0001; nlopt_set_ftol_rel(opt, ftol); - vqe_settings settings; - init_vqe_settings(&settings, qr, cr, NQUBITS, NSHOTS); - - nlopt_result res = nlopt_set_min_objective(opt, vqe_iter_nlopt, (void *)&settings); + nlopt_result res + = nlopt_set_min_objective(opt, vqe_iter_nlopt, (void *)&settings); nlopt_set_lower_bounds1(opt, PARAM_MIN); nlopt_set_upper_bounds1(opt, PARAM_MAX); double best_expectation = 0.0; - res = nlopt_optimize(opt, context.params, &best_expectation); + res = nlopt_optimize(opt, settings.params, &best_expectation); printf("The final expectation: %f\n", best_expectation); diff --git a/include/comms_utils.h b/include/comms_utils.h new file mode 100644 index 0000000..dc7e8ba --- /dev/null +++ b/include/comms_utils.h @@ -0,0 +1,11 @@ +#ifndef CQ_COMMS_UTILS_H +#define CQ_COMMS_UTILS_H + +#include + +bool is_device(); + +#define CQ_PROG_BEGIN() if (!is_device()) { +#define CQ_PROG_END() } + +#endif diff --git a/include/cq.h b/include/cq.h index f4e002a..6d4fae0 100644 --- a/include/cq.h +++ b/include/cq.h @@ -2,12 +2,13 @@ #define CQ_H #include "analog.h" +#include "comms_utils.h" #include "datatypes.h" -#include "env.h" #include "device_ops.h" +#include "env.h" #include "host_ops.h" -#include "qasm_gates.h" #include "kernel_utils.h" +#include "qasm_gates.h" #include "utils.h" #endif diff --git a/include/datatypes.h b/include/datatypes.h index d74886f..d678143 100644 --- a/include/datatypes.h +++ b/include/datatypes.h @@ -27,6 +27,7 @@ typedef struct qubit { } qubit; typedef struct exec { + size_t id; bool exec_init; bool complete; bool halt; @@ -35,6 +36,7 @@ typedef struct exec { size_t completed_shots; size_t expected_shots; size_t nmeasure; + size_t params_size; pthread_mutex_t lock; pthread_cond_t cond_exec_complete; char * fname; diff --git a/include/env.h b/include/env.h index b886172..69f1cc7 100644 --- a/include/env.h +++ b/include/env.h @@ -1,10 +1,11 @@ #ifndef CQ_ENV_H #define CQ_ENV_H -#include -#include #include "datatypes.h" +#include +#include + struct cq_environment { bool initialised; bool finalised; @@ -14,6 +15,16 @@ extern struct cq_environment cq_env; cq_status cq_init(const unsigned int VERBOSITY); +#if CQ_WITH_MPI_COMMS +#include +cq_status cq_init_custom_mpi_comm(MPI_Comm cq_comm, + const unsigned int VERBOSITY); +#endif + cq_status cq_finalise(const unsigned int VERBOSITY); -#endif \ No newline at end of file +// fortran helper -- just don't use it from C +cq_status fort_init_custom_mpi_comm(int * cq_comm, + const unsigned int VERBOSITY); + +#endif diff --git a/include/fortran/cq.f90 b/include/fortran/cq.f90 index 10eb3fe..5b211dc 100644 --- a/include/fortran/cq.f90 +++ b/include/fortran/cq.f90 @@ -50,12 +50,25 @@ module function cq_init(VERBOSITY) result(status) integer :: status end function cq_init + module function cq_init_custom_mpi_comm(cq_comm, VERBOSITY) result(status) + implicit none + integer :: cq_comm + integer, value :: VERBOSITY + integer :: status + end function cq_init_custom_mpi_comm + module function cq_finalise(VERBOSITY) result(status) implicit none integer, value :: VERBOSITY integer :: status end function cq_finalise + module function cq_is_device() result(res) + implicit none + logical :: res + end function cq_is_device + + ! Resource management module function cq_alloc_qubit(qhp) result(status) diff --git a/include/fortran/cqf.h b/include/fortran/cqf.h index 8b6af02..5ece40a 100644 --- a/include/fortran/cqf.h +++ b/include/fortran/cqf.h @@ -1,3 +1,7 @@ -#define CQ_REGISTER_KERNEL(fname, reg) \ - status = cq_register_fort_kernel(fname, reg); \ - if (status == 0) return +#define CQ_REGISTER_KERNEL(fname, reg) \ + status = cq_register_fort_kernel(fname, reg); \ + if (status == 0) return + +#define CQ_PROG_BEGIN() \ + if (.NOT.cq_is_device()) then +#define CQ_PROG_END() end if diff --git a/include/host_ops.h b/include/host_ops.h index 8be1e8d..f3bd70e 100644 --- a/include/host_ops.h +++ b/include/host_ops.h @@ -16,61 +16,141 @@ cq_status free_qureg(qubit ** qrp); // Executors -cq_status s_qrun(qkern kernel, qubit * qrp, const size_t NQUBITS, cstate * const crp, - const size_t NMEASURE); - -cq_status a_qrun(qkern kernel, qubit * qrp, const size_t NQUBITS, cstate * const crp, - const size_t NMEASURE, cq_exec * const ehp); - -cq_status sm_qrun(qkern kernel, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, const size_t NSHOTS); - -cq_status am_qrun(qkern kernel, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, const size_t NSHOTS, - cq_exec * const ehp); - -cq_status sb_qrun(qkern kernel, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, const backend_id BE); - -cq_status ab_qrun(qkern kernel, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, const backend_id BE, - cq_exec * const ehp); - -cq_status smb_qrun(qkern kernel, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, const size_t NSHOTS, - const backend_id BE); - -cq_status amb_qrun(qkern kernel, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, const size_t NSHOTS, - const backend_id BE, cq_exec * const ehp); - -cq_status sp_qrun(pqkern kernel, void * kernpar, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE); - -cq_status ap_qrun(pqkern kernel, void * kernpar, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, cq_exec * const ehp); - -cq_status smp_qrun(pqkern kernel, void * kernpar, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, const size_t NSHOTS); - -cq_status amp_qrun(pqkern kernel, void * kernpar, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, const size_t NSHOTS, - cq_exec * const ehp); - -cq_status sbp_qrun(pqkern kernel, void * kernpar, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, const backend_id BE); - -cq_status abp_qrun(pqkern kernel, void * kernpar, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, - const backend_id BE, cq_exec * const ehp); - -cq_status smbp_qrun(pqkern kernel, void * kernpar, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, const size_t NSHOTS, - const backend_id BE); - -cq_status ambp_qrun(pqkern kernel, void * kernpar, qubit * qrp, const size_t NQUBITS, - cstate * const crp, const size_t NMEASURE, const size_t NSHOTS, - const backend_id BE, cq_exec * const ehp); +cq_status s_qrun(qkern kernel, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE); + +cq_status a_qrun(qkern kernel, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + cq_exec * const ehp); + +cq_status sm_qrun(qkern kernel, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const size_t NSHOTS); + +cq_status am_qrun(qkern kernel, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const size_t NSHOTS, + cq_exec * const ehp); + +cq_status sb_qrun(qkern kernel, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const backend_id BE); + +cq_status ab_qrun(qkern kernel, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const backend_id BE, + cq_exec * const ehp); + +cq_status smb_qrun(qkern kernel, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const size_t NSHOTS, + const backend_id BE); + +cq_status amb_qrun(qkern kernel, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const size_t NSHOTS, + const backend_id BE, + cq_exec * const ehp); + +// NOTE: requires modification of CQ spec +cq_status sp_qrun(pqkern kernel, + void * kernpar, + const size_t KERNPAR_SIZE, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE); + +// NOTE: requires modification of CQ spec +cq_status ap_qrun(pqkern kernel, + void * kernpar, + const size_t KERNPAR_SIZE, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + cq_exec * const ehp); + +// NOTE: requires modification of CQ spec +cq_status smp_qrun(pqkern kernel, + void * kernpar, + const size_t KERNPAR_SIZE, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const size_t NSHOTS); + +// NOTE: requires modification of CQ spec +cq_status amp_qrun(pqkern kernel, + void * kernpar, + const size_t KERNPAR_SIZE, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const size_t NSHOTS, + cq_exec * const ehp); + +cq_status sbp_qrun(pqkern kernel, + void * kernpar, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const backend_id BE); + +cq_status abp_qrun(pqkern kernel, + void * kernpar, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const backend_id BE, + cq_exec * const ehp); + +cq_status smbp_qrun(pqkern kernel, + void * kernpar, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const size_t NSHOTS, + const backend_id BE); + +cq_status ambp_qrun(pqkern kernel, + void * kernpar, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const size_t NSHOTS, + const backend_id BE, + cq_exec * const ehp); // Synchronisation @@ -80,4 +160,4 @@ cq_status wait_qrun(cq_exec * const ehp); cq_status halt_qrun(cq_exec * const ehp); -#endif \ No newline at end of file +#endif diff --git a/include/utils.h b/include/utils.h index f7cb9d2..a82c6ca 100644 --- a/include/utils.h +++ b/include/utils.h @@ -5,18 +5,16 @@ void init_creg(const size_t LENGTH, const cstate INIT_VAL, cstate * cr); -void report_results( - cstate const * const CR, - const size_t NMEASURE, - const size_t NSHOTS); +void report_results(cstate const * const CR, + const size_t NMEASURE, + const size_t NSHOTS); -#define HANDLE_CQ_ERROR(x) \ - { \ - if (x == CQ_ERROR) \ - { \ - printf(" From: %s\n", __func__); \ - return CQ_ERROR; \ - } \ - }; +#define HANDLE_CQ_ERROR(x) \ + { \ + if (x == CQ_ERROR) { \ + printf(" From: %s\n", __func__); \ + return CQ_ERROR; \ + } \ + }; -#endif \ No newline at end of file +#endif diff --git a/src/device/control.c b/src/device/control.c index acd4628..02b384b 100644 --- a/src/device/control.c +++ b/src/device/control.c @@ -6,8 +6,10 @@ #include "utils.h" #include "kernel_utils.h" #include "resources.h" + #include "quest/include/environment.h" + cq_status (*control_registry[8])(void *) = { initialise_simulator, abort_current_kernel, @@ -30,7 +32,7 @@ cq_status initialise_simulator(void * par) { printf("Initialising QuEST.\n"); } - initQuESTEnv(); + init_quest_env(); if (*pVERBOSITY > 0) { reportQuESTEnv(); @@ -64,6 +66,10 @@ cq_status finalise_simulator(void * par) { finalizeQuESTEnv(); + if (*pVERBOSITY > 0) { + printf("QuEST finalised\n"); + } + // isQuESTEnvInit returns 1 for true, 0 for false if (!isQuESTEnvInit()) { status = CQ_SUCCESS; diff --git a/src/host-device/CMakeLists.txt b/src/host-device/CMakeLists.txt index df2447f..7f5392a 100644 --- a/src/host-device/CMakeLists.txt +++ b/src/host-device/CMakeLists.txt @@ -1,5 +1,6 @@ +add_subdirectory(comms) + target_sources(cq-simbe PRIVATE - comms.c kernel_utils.c -) \ No newline at end of file +) diff --git a/src/host-device/comms.h b/src/host-device/comms.h index 537677d..19b96b4 100644 --- a/src/host-device/comms.h +++ b/src/host-device/comms.h @@ -1,13 +1,22 @@ -#ifndef HOST_DEVICE_COMMS_H -#define HOST_DEVICE_COMMS_H +#ifndef CQ_HOST_DEVICE_COMMS_H +#define CQ_HOST_DEVICE_COMMS_H -#include -#include +#include "comms_utils.h" #include "datatypes.h" #include "src/host/opcodes.h" +#include +#include + #define __CQ_DEVICE_QUEUE_SIZE__ 16 +#define RUN_HOST_ONLY() \ + { \ + if (is_device()) { \ + return CQ_SUCCESS; \ + } \ + } + struct dev_link { bool run_device; pthread_t device_thread; @@ -15,7 +24,7 @@ struct dev_link { bool device_busy; pthread_cond_t cond_device_busy; - + size_t num_ops; pthread_cond_t cond_queue_empty; pthread_cond_t cond_queue_full; @@ -36,21 +45,32 @@ extern struct dev_link dev_ctrl; int initialise_device(const unsigned int VERBOSITY); -size_t host_send_ctrl_op(const enum ctrl_code OP, void * ctrl_params); +#if CQ_WITH_MPI_COMMS +#include +int initialise_device_with_custom_mpi_comm(MPI_Comm cq_comm, + const unsigned int VERBOSITY); +#endif -size_t host_sync_exec(cq_exec * const ehp); +size_t host_send_ctrl_op(const enum ctrl_code OP, void * ctrl_params); -size_t host_wait_exec(cq_exec * const ehp); +size_t host_wait_all_ops(void); -void host_request_halt(cq_exec * const ehp); +size_t device_sync_exec(const cq_status STATUS, + const size_t SHOT, + cstate const * const RESULT, + cq_exec * ehp); -size_t host_wait_all_ops(); +void host_device_sync_comms(void); -size_t device_sync_exec(const cq_status STATUS, const size_t SHOT, - cstate const * const RESULT, cq_exec * ehp); +int finalise_device(const unsigned int VERBOSITY); -void * device_control_thread(void *); +/// generates and assigns id to the executor that is used for host-device +/// communication. +/// @return new executor id +size_t assign_exec_id(void); -int finalise_device(const unsigned int VERBOSITY); +/// +/// initialises correct QuEST environment depending on the build opotions. +void init_quest_env(void); -#endif \ No newline at end of file +#endif diff --git a/src/host-device/comms/CMakeLists.txt b/src/host-device/comms/CMakeLists.txt new file mode 100644 index 0000000..8c49b0d --- /dev/null +++ b/src/host-device/comms/CMakeLists.txt @@ -0,0 +1,16 @@ +target_sources(cq-simbe + PRIVATE + comms_core.c +) + +if (BUILD_WITH_MPI) + target_sources(cq-simbe + PRIVATE + comms_mpi.c + ) +else() + target_sources(cq-simbe + PRIVATE + comms.c + ) +endif() diff --git a/src/host-device/comms/comms.c b/src/host-device/comms/comms.c new file mode 100644 index 0000000..2b1090f --- /dev/null +++ b/src/host-device/comms/comms.c @@ -0,0 +1,35 @@ +#include "src/host-device/comms.h" + +#include "comms_core.h" + +#include "quest/include/environment.h" + +#include + +int initialise_device(const unsigned int VERBOSITY) { + return serial_initialise_device(VERBOSITY); +} + +size_t host_send_ctrl_op(const enum ctrl_code OP, void * ctrl_params) { + return serial_host_send_ctrl_op(OP, ctrl_params); +} + +size_t host_wait_all_ops(void) { + return serial_host_wait_all_ops(); +} + +void host_device_sync_comms(void) { + serial_host_device_sync_comms(); +} + +int finalise_device(const unsigned int VERBOSITY) { + return serial_finalise_device(VERBOSITY); +} + +bool is_device(void) { + return false; +} + +void init_quest_env(void) { + initQuESTEnv(); +} diff --git a/src/host-device/comms.c b/src/host-device/comms/comms_core.c similarity index 67% rename from src/host-device/comms.c rename to src/host-device/comms/comms_core.c index f81045e..73980a6 100644 --- a/src/host-device/comms.c +++ b/src/host-device/comms/comms_core.c @@ -1,14 +1,15 @@ -#include -#include +#include "comms_core.h" + +#include "../comms.h" +#include "src/device/control.h" +#include "src/host/opcodes.h" + #include #include -#include "datatypes.h" -#include "kernel_utils.h" -#include "comms.h" -#include "src/host/opcodes.h" -#include "src/device/control.h" -int initialise_device(const unsigned int VERBOSITY) { +static size_t global_exec_id_counter = -1; + +int init_device_controls(const unsigned int VERBOSITY) { if (VERBOSITY > 0) { printf("Initialising device.\n"); } @@ -30,14 +31,30 @@ int initialise_device(const unsigned int VERBOSITY) { pthread_create(&dev_ctrl.device_thread, NULL, &device_control_thread, NULL); - unsigned int verbosity = VERBOSITY; - host_send_ctrl_op(CQ_CTRL_INIT, &verbosity); - host_wait_all_ops(); + return 0; +} + +void stop_device(void) { + pthread_mutex_lock(&dev_ctrl.device_lock); + dev_ctrl.run_device = false; + pthread_mutex_unlock(&dev_ctrl.device_lock); +} + +int finalise_device_controls(const unsigned int VERBOSITY) { + pthread_join(dev_ctrl.device_thread, NULL); + + dev_ctrl.device_busy = false; + + pthread_cond_destroy(&dev_ctrl.cond_device_busy); + pthread_cond_destroy(&dev_ctrl.cond_queue_empty); + pthread_cond_destroy(&dev_ctrl.cond_queue_full); + pthread_mutex_destroy(&dev_ctrl.device_lock); return 0; } -size_t host_send_ctrl_op(const enum ctrl_code OP, void * ctrl_params) { +// NOTE: aka host_send_ctrl_op from OG comm.c +size_t insert_op(const enum ctrl_code OP, void * ctrl_params) { pthread_mutex_lock(&dev_ctrl.device_lock); while (dev_ctrl.num_ops >= __CQ_DEVICE_QUEUE_SIZE__) { @@ -50,7 +67,7 @@ size_t host_send_ctrl_op(const enum ctrl_code OP, void * ctrl_params) { dev_ctrl.op_params_buffer[dev_ctrl.next_op_in] = ctrl_params; ++dev_ctrl.num_ops; - // It's a ring buffer! + // It's a ring buffer! // advance next_op_in then mod out buffer size ++dev_ctrl.next_op_in; dev_ctrl.next_op_in %= __CQ_DEVICE_QUEUE_SIZE__; @@ -61,7 +78,10 @@ size_t host_send_ctrl_op(const enum ctrl_code OP, void * ctrl_params) { return dev_ctrl.num_ops; } -size_t host_sync_exec(cq_exec * const ehp) { +size_t comms_exec_sync(cq_exec * const ehp) { + if (ehp == NULL) { + return 0; + } size_t completed_shots = 0; pthread_mutex_lock(&(ehp->lock)); completed_shots = ehp->completed_shots; @@ -69,71 +89,41 @@ size_t host_sync_exec(cq_exec * const ehp) { return completed_shots; } -size_t host_wait_exec(cq_exec * const ehp) { +size_t comms_exec_wait(cq_exec * const ehp) { + if (ehp == NULL) { + return 0; + } + pthread_mutex_lock(&(ehp->lock)); while (!ehp->complete) { - pthread_cond_wait( - &(ehp->cond_exec_complete), - &(ehp->lock) - ); + pthread_cond_wait(&(ehp->cond_exec_complete), &(ehp->lock)); } pthread_mutex_unlock(&(ehp->lock)); return ehp->completed_shots; } -void host_request_halt(cq_exec * const ehp) { +void comms_exec_halt(cq_exec * const ehp) { + if (ehp == NULL) { + return; + } + pthread_mutex_lock(&(ehp->lock)); ehp->halt = true; pthread_mutex_unlock(&(ehp->lock)); return; } -size_t host_wait_all_ops() { +// NOTE: aka host_wait_all_ops from OG comm.c +size_t device_wait_all_ops(void) { pthread_mutex_lock(&dev_ctrl.device_lock); - while(dev_ctrl.num_ops > 0 || dev_ctrl.device_busy) { - pthread_cond_wait( - &dev_ctrl.cond_device_busy, - &dev_ctrl.device_lock - ); + while (dev_ctrl.num_ops > 0 || dev_ctrl.device_busy) { + pthread_cond_wait(&dev_ctrl.cond_device_busy, &dev_ctrl.device_lock); } + pthread_mutex_unlock(&dev_ctrl.device_lock); return dev_ctrl.num_ops; } -size_t device_sync_exec(const cq_status STATUS, const size_t SHOT, -cstate const * const RESULT, cq_exec * ehp) { - pthread_mutex_lock(&ehp->lock); - - if (STATUS == CQ_EARLY_SUCCESS) { - // generally speaking we should respect the kernel-provided - // status code, but CQ_EARLY_SUCCESS really means CQ_SUCCESS - // but needed to be != so we could break execution - ehp->status = CQ_SUCCESS; - } else { - ehp->status = STATUS; - } - - ehp->completed_shots += 1; - - // copy local result register to exec - cstate * dest_creg = ehp->creg + SHOT * ehp->nmeasure; - memcpy(dest_creg, RESULT, ehp->nmeasure * sizeof(cstate)); - - // check if the whole execution is done - if ( - ehp->completed_shots == ehp->expected_shots - || STATUS != CQ_SUCCESS - || ehp->halt - ) { - ehp->complete = true; - pthread_cond_signal(&(ehp->cond_exec_complete)); - } - - pthread_mutex_unlock(&ehp->lock); - - return SHOT; -} - void * device_control_thread(void * par) { enum ctrl_code current_op = CQ_CTRL_IDLE; void * current_op_params = NULL; @@ -142,16 +132,17 @@ void * device_control_thread(void * par) { while (dev_ctrl.run_device) { pthread_mutex_lock(&dev_ctrl.device_lock); - while(dev_ctrl.num_ops <= 0) { + while (dev_ctrl.num_ops <= 0) { // wait for a new op to be posted dev_ctrl.device_busy = false; pthread_cond_signal(&dev_ctrl.cond_device_busy); pthread_cond_wait(&dev_ctrl.cond_queue_empty, &dev_ctrl.device_lock); } - + dev_ctrl.device_busy = true; - // take the next op and params out of the dev_ctrl buffer, and then tidy up the dev_ctrl buffer + // take the next op and params out of the dev_ctrl buffer, and then tidy up + // the dev_ctrl buffer current_op = dev_ctrl.op_buffer[dev_ctrl.next_op_out]; current_op_params = dev_ctrl.op_params_buffer[dev_ctrl.next_op_out]; dev_ctrl.op_buffer[dev_ctrl.next_op_out] = CQ_CTRL_IDLE; @@ -167,38 +158,102 @@ void * device_control_thread(void * par) { pthread_mutex_unlock(&dev_ctrl.device_lock); control_registry[current_op](current_op_params); - } - + } + pthread_mutex_unlock(&dev_ctrl.device_lock); return NULL; } -int finalise_device(const unsigned int VERBOSITY) { +size_t device_sync_exec(const cq_status STATUS, + const size_t SHOT, + cstate const * const RESULT, + cq_exec * ehp) { + pthread_mutex_lock(&ehp->lock); + + if (STATUS == CQ_EARLY_SUCCESS) { + // generally speaking we should respect the kernel-provided + // status code, but CQ_EARLY_SUCCESS really means CQ_SUCCESS + // but needed to be != so we could break execution + ehp->status = CQ_SUCCESS; + } else { + ehp->status = STATUS; + } + + ehp->completed_shots += 1; + + // copy local result register to exec + cstate * dest_creg = ehp->creg + SHOT * ehp->nmeasure; + memcpy(dest_creg, RESULT, ehp->nmeasure * sizeof(cstate)); + + // check if the whole execution is done + if (ehp->completed_shots == ehp->expected_shots || STATUS != CQ_SUCCESS + || ehp->halt) { + ehp->complete = true; + pthread_cond_signal(&(ehp->cond_exec_complete)); + } + + pthread_mutex_unlock(&ehp->lock); + + return SHOT; +} + +size_t assign_exec_id(void) { + ++global_exec_id_counter; + global_exec_id_counter %= __CQ_DEVICE_QUEUE_SIZE__; + return global_exec_id_counter; +} + +size_t serial_host_send_ctrl_op(const enum ctrl_code OP, void * ctrl_params) { + switch (OP) { + case CQ_CTRL_SYNC_EXEC: { + return comms_exec_sync(ctrl_params); + break; + } + case CQ_CTRL_WAIT_EXEC: { + return comms_exec_wait(ctrl_params); + break; + } + case CQ_CTRL_ABORT: { + comms_exec_halt(ctrl_params); + return 0; + break; + } + default: { + break; + } + } + return insert_op(OP, ctrl_params); +} + +int serial_initialise_device(const unsigned int VERBOSITY) { + init_device_controls(VERBOSITY); + unsigned int verbosity = VERBOSITY; + host_send_ctrl_op(CQ_CTRL_INIT, &verbosity); + host_wait_all_ops(); + return 0; +} + +size_t serial_host_wait_all_ops(void) { + return device_wait_all_ops(); +} + +void serial_host_device_sync_comms(void) {} + +int serial_finalise_device(const unsigned int VERBOSITY) { // Politely wait for the device to finish its current business // otherwise setting dev_ctrl.run_device = false might break // some stuff, and this should only be called in cq_finalise() - host_wait_all_ops(); + serial_host_wait_all_ops(); if (VERBOSITY > 0) { printf("Finalising device.\n"); } - pthread_mutex_lock(&dev_ctrl.device_lock); - dev_ctrl.run_device = false; - pthread_mutex_unlock(&dev_ctrl.device_lock); - + stop_device(); unsigned int verbosity = VERBOSITY; - host_send_ctrl_op(CQ_CTRL_FINALISE, &verbosity); - - pthread_join(dev_ctrl.device_thread, NULL); - - dev_ctrl.device_busy = false; - - pthread_cond_destroy(&dev_ctrl.cond_device_busy); - pthread_cond_destroy(&dev_ctrl.cond_queue_empty); - pthread_cond_destroy(&dev_ctrl.cond_queue_full); - pthread_mutex_destroy(&dev_ctrl.device_lock); + serial_host_send_ctrl_op(CQ_CTRL_FINALISE, &verbosity); + finalise_device_controls(VERBOSITY); return 0; -} \ No newline at end of file +} diff --git a/src/host-device/comms/comms_core.h b/src/host-device/comms/comms_core.h new file mode 100644 index 0000000..3b1745f --- /dev/null +++ b/src/host-device/comms/comms_core.h @@ -0,0 +1,37 @@ +#ifndef CQ_HOST_DEVICE_COMMS_CORE_H +#define CQ_HOST_DEVICE_COMMS_CORE_H + +#include "datatypes.h" +#include "src/host/opcodes.h" + +#include + +int init_device_controls(const unsigned int VERBOSITY); + +void stop_device(void); + +int finalise_device_controls(const unsigned int VERBOSITY); + +size_t insert_op(const enum ctrl_code OP, void * ctrl_params); + +size_t comms_exec_sync(cq_exec * const ehp); + +size_t comms_exec_wait(cq_exec * const ehp); + +void comms_exec_halt(cq_exec * const ehp); + +size_t device_wait_all_ops(void); + +void * device_control_thread(void *); + +size_t serial_host_send_ctrl_op(const enum ctrl_code OP, void * ctrl_params); + +int serial_initialise_device(const unsigned int VERBOSITY); + +size_t serial_host_wait_all_ops(void); + +void serial_host_device_sync_comms(void); + +int serial_finalise_device(const unsigned int VERBOSITY); + +#endif diff --git a/src/host-device/comms/comms_mpi.c b/src/host-device/comms/comms_mpi.c new file mode 100644 index 0000000..68e1d3f --- /dev/null +++ b/src/host-device/comms/comms_mpi.c @@ -0,0 +1,1275 @@ +#include "comms_mpi.h" +#include +#include "comms_core.h" +#include "src/host-device/comms.h" + +#include "datatypes.h" +#include "src/host/opcodes.h" + +#ifndef CQ_CONF_QUEST_WITH_MPI +#include "quest/include/environment.h" +#endif + +#if CQ_CONF_QUEST_WITH_MPI +#include "quest/include/experimental.h" +#endif + +#include + +#include +#include +#include +#include +#include +#include + +// ---------------------------------------------------------------------------- +// Macros +// ---------------------------------------------------------------------------- +// TODO: change these to be variables rather than macros +// i.e. CQ_MPI_HOST_RANK is the one specified by user +// CQ_MPI_DEVICE RANK is also specified by user or CQ_MPI_HOST_RANK + 1 or +// CQ_MPI_HOST_RANK if running with only 1 proc +// +// so default values for host_rank = 0, device_rank = host_rank + 1 +// but user can pass as args +// #define CQ_MPI_HOST_RANK 0 +// #define CQ_MPI_DEVICE_RANK 1 +#define CQ_MPI_DEVICE_MASTER_RANK 0 +#define CQ_MPI_WORLD_COMMS_TAG 0 +#define CQ_MPI_SUBCOMMS_TAG 0 + +#define CQ_MPI_RUNTIME_ERROR -4 +#define CQ_MPI_MALLOC_ERROR -5 + +static MPI_Comm CQ_MPI_COMM_WORLD = MPI_COMM_NULL; +static MPI_Comm CQ_MPI_SPLIT_COMM = MPI_COMM_NULL; +static bool owns_mpi_comm = false; + +static int CQ_MPI_HOST_RANK = 0; +static int CQ_MPI_DEVICE_RANK = 0; + +struct cq_mpi_env { + bool is_init; + int rank; + int subcomm_rank; + unsigned int verbosity; + int world_size; +}; + +static cq_exec * executor_handles[__CQ_DEVICE_QUEUE_SIZE__]; +static size_t num_active_executors = 0; +static struct cq_mpi_env mpi_env = { + .is_init = false, + .rank = -1, + .subcomm_rank = -1, + .verbosity = 0, + .world_size = 0, +}; + +struct communicator { + bool comm_busy; + pthread_t device_comm_thread; + pthread_cond_t cond_comm_busy; + pthread_mutex_t comm_lock; +}; + +static struct communicator dev_comm = { 0 }; + +static char debug_worker_name[64] = { 0 }; + +static void cq_log(const char * format, ...) { +#ifdef DEBUG_MODE + va_list(args); + va_start(args, format); + vprintf(format, args); +#endif +} + +static bool cq_validate_mpi_rank(const int rank) { + if (rank < 0 || rank >= mpi_env.world_size) { + cq_log( + "passed incorrect mpi rank with value either < 0 or >= world_size\n"); + return false; + } + return true; +} + +static bool is_serial() { + return mpi_env.world_size == 1 || !mpi_env.is_init; +} + +int initialise_device_with_custom_mpi_comm(MPI_Comm cq_comm, + const unsigned int VERBOSITY) { + MPI_Comm_dup(cq_comm, &CQ_MPI_COMM_WORLD); + return initialise_device(VERBOSITY); +} + +int initialise_device(const unsigned int VERBOSITY) { + if (!mpi_env.is_init) { + mpi_env.is_init = true; + init_host_device_mpi(VERBOSITY); + } + + if (is_serial()) { + return serial_initialise_device(VERBOSITY); + } + + RUN_HOST_ONLY(); + unsigned int verbosity = VERBOSITY; + host_send_ctrl_op(CQ_CTRL_INIT, &verbosity); + host_wait_all_ops(); + return 0; +} + +size_t host_send_ctrl_op(const enum ctrl_code OP, void * params) { + if (is_serial()) { + return serial_host_send_ctrl_op(OP, params); + } + + const int device_rank = CQ_MPI_DEVICE_RANK; + cq_log("%s [send_ctrl_op]: sending %s...\n", get_comm_source(), + op_to_str(OP)); + int op_comm_buffer = (int)OP; + MPI_Ssend(&op_comm_buffer, 1, MPI_INT, device_rank, CQ_MPI_WORLD_COMMS_TAG, + CQ_MPI_COMM_WORLD); + // send params based on OP + host_comm_params(OP, params); + cq_log("%s [send_ctrl_op]: sent %s\n", get_comm_source(), op_to_str(OP)); + return 0; +} + +size_t host_wait_all_ops(void) { + if (is_serial()) { + return serial_host_wait_all_ops(); + } + + cq_log("%s [wait_all_ops]: waiting...\n", get_comm_source()); + host_send_ctrl_op(CQ_CTRL_WAIT, NULL); + size_t num_ops; + const int device_rank = CQ_MPI_DEVICE_RANK; + MPI_Status status; + MPI_Recv(&num_ops, 1, MPI_UINT64_T, device_rank, CQ_MPI_WORLD_COMMS_TAG, + CQ_MPI_COMM_WORLD, &status); + cq_log("%s [wait_all_ops]: all ops completed (num_ops: %zu)\n", + get_comm_source(), num_ops); + return 0; +} + +void host_device_sync_comms(void) { + if (is_serial()) { + return serial_host_device_sync_comms(); + } + + if (mpi_env.rank != CQ_MPI_HOST_RANK) { + cq_log("%s [final_sync]: simply waiting\n", get_comm_source()); + device_wait_comms(); + } +} + +int finalise_device(const unsigned int VERBOSITY) { + if (is_serial()) { + int res = serial_finalise_device(VERBOSITY); + finalise_host_device_mpi(VERBOSITY); + return res; + } + + host_device_sync_comms(); + + if (mpi_env.rank == CQ_MPI_HOST_RANK) { + host_wait_all_ops(); + + if (VERBOSITY > 0) { + printf("Finalising device.\n"); + } + + unsigned int verbosity = VERBOSITY; + host_send_ctrl_op(CQ_CTRL_FINALISE, &verbosity); + } + + finalise_host_device_mpi(VERBOSITY); + + return 0; +} + +bool is_device(void) { + if (!mpi_env.is_init) { + return false; + } + + if (mpi_env.rank < 0) { + cq_log("%s [is_device]: rank is < 0 => MPI not initialised. Exiting!\n", + get_comm_source()); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_RUNTIME_ERROR); + } + + return mpi_env.rank != CQ_MPI_HOST_RANK; +} + +void init_quest_env(void) { +#if CQ_WITH_MPI_COMMS && CQ_CONF_QUEST_WITH_MPI + if (is_serial()) { + initCustomMpiQuESTEnv(0, 1, 0, 0); + } else if (is_quantum_worker()) { + initCustomMpiCommQuESTEnv(CQ_MPI_SPLIT_COMM, 0, 0); + } else { + } +#endif +#ifndef CQ_CONF_QUEST_WITH_MPI + initQuESTEnv(); +#endif +} + +// ---------------------------------------------------------------------------- +// Device Comm Ops +// ---------------------------------------------------------------------------- + +void init_host_device_mpi(const unsigned int VERBOSITY) { + mpi_env.verbosity = VERBOSITY; + if (VERBOSITY > 0) { + cq_log("Initialising MPI.\n"); + } + + if (CQ_MPI_COMM_WORLD == MPI_COMM_NULL) { + owns_mpi_comm = true; + + int provided; + MPI_Init_thread(NULL, NULL, MPI_THREAD_MULTIPLE, &provided); + + if (provided != MPI_THREAD_MULTIPLE) { + cq_log( + "Failed setting up multithreading with MPI. Requested " + "MPI_THREAD_MULTIPLE, given: %d\n", + provided); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_RUNTIME_ERROR); + } + + MPI_Comm_dup(MPI_COMM_WORLD, &CQ_MPI_COMM_WORLD); + } + + MPI_Comm_size(CQ_MPI_COMM_WORLD, &mpi_env.world_size); + MPI_Comm_rank(CQ_MPI_COMM_WORLD, &mpi_env.rank); + + cq_log("CQ comm size: %d\n", mpi_env.world_size); + cq_log("%s in CQ subcomm I'm rank %d\n", get_comm_source(), mpi_env.rank); + + // TODO: for multi-device check that: + // (nproc - 1) == n_device * power of 2 + validate_nproc(mpi_env.world_size); + + if (is_serial()) { + MPI_Barrier(CQ_MPI_COMM_WORLD); + MPI_Comm_dup(CQ_MPI_COMM_WORLD, &CQ_MPI_SPLIT_COMM); + return; + } + +#if CQ_CONF_QUEST_WITH_MPI + const int QUANTUM_WORKER = mpi_env.rank >= CQ_MPI_DEVICE_RANK; + MPI_Barrier(CQ_MPI_COMM_WORLD); + MPI_Comm_split(CQ_MPI_COMM_WORLD, QUANTUM_WORKER, mpi_env.rank, + &CQ_MPI_SPLIT_COMM); + MPI_Comm_rank(CQ_MPI_SPLIT_COMM, &mpi_env.subcomm_rank); + + int quest_size; + MPI_Comm_size(CQ_MPI_SPLIT_COMM, &quest_size); + cq_log("QuEST comm size: %d\n", quest_size); + + cq_log("%s in subcomm I'm rank %d\n", get_comm_source(), + mpi_env.subcomm_rank); + + sprintf(debug_worker_name, "Quantum Worker [%d]:\t\t", mpi_env.subcomm_rank); +#endif + + if (VERBOSITY > 0) { + cq_log("Initialised MPI.\n"); + } + + if (mpi_env.rank != CQ_MPI_HOST_RANK) { + device_init_comms(VERBOSITY); + for (size_t i = 0; i < __CQ_DEVICE_QUEUE_SIZE__; ++i) { + executor_handles[i] = NULL; + } + } +} + +void finalise_host_device_mpi(const unsigned int VERBOSITY) { + if (VERBOSITY > 0) { + cq_log("%s Finalising MPI.\n", get_comm_source()); + } + if (owns_mpi_comm) { + MPI_Barrier(CQ_MPI_COMM_WORLD); + MPI_Finalize(); + } + if (VERBOSITY > 0) { + cq_log("%s Finalised MPI.\n", get_comm_source()); + } +} + +void device_init_comms(const unsigned int VERBOSITY) { + if (VERBOSITY > 0) { + cq_log( + "%s [device_init_comms]: setting up on-device communication thread.\n", + get_comm_source()); + } + dev_ctrl.run_device = true; + dev_comm.comm_busy = true; + + pthread_mutex_init(&dev_comm.comm_lock, NULL); + pthread_cond_init(&dev_comm.cond_comm_busy, NULL); + pthread_create(&dev_comm.device_comm_thread, NULL, &device_listen, NULL); + + if (VERBOSITY > 0) { + cq_log("%s [device_init_comms]: on-device communication thread set up.\n", + get_comm_source()); + } +} + +void device_finalise_comms(const unsigned int VERBOSITY) { + if (VERBOSITY > 0) { + cq_log( + "%s [device_finalise_comms]: finalising on-device communication " + "thread.\n", + get_comm_source()); + } + + pthread_join(dev_comm.device_comm_thread, NULL); + dev_comm.comm_busy = false; + pthread_cond_destroy(&dev_comm.cond_comm_busy); + pthread_mutex_destroy(&dev_comm.comm_lock); + if (VERBOSITY > 0) { + cq_log( + "%s [device_finalise_comms]: on-device communication thread closed.\n", + get_comm_source()); + } +} + +void * device_listen(void * args) { + // run_device set to FALSE when OP == CQ_CTRL_FINALISE + cq_log("%s started listening...\n", get_comm_source()); + while (dev_ctrl.run_device) { + pthread_mutex_lock(&dev_comm.comm_lock); + dev_comm.comm_busy = true; + pthread_cond_signal(&dev_comm.cond_comm_busy); + pthread_mutex_unlock(&dev_comm.comm_lock); + + enum ctrl_code OP; + int op_comm_buffer; + + cq_log("%s [device_listen]: receiving OP...\n", get_comm_source()); +#if CQ_CONF_QUEST_WITH_MPI + // device master (rank 0) gets from host from different comm + if (mpi_env.subcomm_rank == CQ_MPI_DEVICE_MASTER_RANK) { +#endif + + const int host_rank = CQ_MPI_HOST_RANK; + MPI_Status status; + MPI_Recv(&op_comm_buffer, 1, MPI_INT, host_rank, CQ_MPI_WORLD_COMMS_TAG, + CQ_MPI_COMM_WORLD, &status); +#if CQ_CONF_QUEST_WITH_MPI + // rank 0 brodcast to rest of q-workers then all do the dispatch + } + cq_log("%s [device_listen]: Starting Bcast\n", get_comm_source()); + MPI_Bcast(&op_comm_buffer, 1, MPI_INT, CQ_MPI_DEVICE_MASTER_RANK, + CQ_MPI_SPLIT_COMM); + cq_log("%s [device_listen]: Finished Bcast\n", get_comm_source()); +#endif + + OP = (enum ctrl_code)op_comm_buffer; + cq_log("%s [device_listen]: received %s\n", get_comm_source(), + op_to_str(OP)); + + device_dispatch_ctrl_op(OP); + + pthread_mutex_lock(&dev_comm.comm_lock); + dev_comm.comm_busy = false; + pthread_cond_signal(&dev_comm.cond_comm_busy); + pthread_mutex_unlock(&dev_comm.comm_lock); + } + cq_log("%s closing connection.\n", get_comm_source()); + + device_wait_all_ops(); + finalise_device_controls(mpi_env.verbosity); + + return NULL; +} + +void device_dispatch_ctrl_op(const enum ctrl_code OP) { + switch (OP) { + case CQ_CTRL_INIT: { + init_device_controls(mpi_env.verbosity); + insert_op(OP, &mpi_env.verbosity); + device_wait_all_ops(); + break; + } + case CQ_CTRL_FINALISE: { + // we wait until worker is done and cleanup. + device_wait_all_ops(); + for (size_t i = 0; i < __CQ_DEVICE_QUEUE_SIZE__; ++i) { + if (executor_handles[i] != NULL) { + cq_log( + "%s [dispatch][FINALISE]: executor handle with id: %zu is still " + "active. Something went wrong!\n", + get_comm_source(), i); + } + } + + // this is like finalise_device in original comms.c + insert_op(OP, &mpi_env.verbosity); + device_wait_all_ops(); + stop_device(); + break; + } + case CQ_CTRL_ALLOC: { + // Alloc is blocking: we get params, run allocation and + // send back the updated params. + // Also, because it's blocking I don't need to worry about + // params lifetime (from the worker perspective) + const int host_rank = CQ_MPI_HOST_RANK; + device_alloc_params params = { 0 }; + recv_alloc_params(¶ms, host_rank); + insert_op(OP, ¶ms); + cq_log("%s [dispatch][ALLOC]: inserted op\n", get_comm_source()); + device_wait_all_ops(); + send_alloc_params(¶ms, host_rank); + break; + } + case CQ_CTRL_DEALLOC: { + // Same as Alloc + const int host_rank = CQ_MPI_HOST_RANK; + device_alloc_params params = { 0 }; + recv_alloc_params(¶ms, host_rank); + insert_op(OP, ¶ms); + device_wait_all_ops(); + send_alloc_params(¶ms, host_rank); + break; + } + case CQ_CTRL_RUN_QKERNEL: { + if (num_active_executors >= __CQ_DEVICE_QUEUE_SIZE__) { + cq_log( + "%s [dispatch][RUN_QKERNEL]: You have oversubsribed the executor " + "queue. We allow up to %d " + "concurrent executors per device. Exiting", + get_comm_source(), __CQ_DEVICE_QUEUE_SIZE__); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_RUNTIME_ERROR); + } + const int host_rank = CQ_MPI_HOST_RANK; + cq_exec * tmp_exec = NULL; + recv_exec_params(&tmp_exec, host_rank); + // NOTE: this can be done here to free old one + // and allocate new one rather than in wait_exec + // device_free_exec(&executor_handles[tmp_exec->id]); + // actually if executor_handles[id] != NULL then it didn't finish and we + // should wait here until we are done. + // + // NOTE: 2 + // Now the question is should we block? + // 1. if we block here and wait for all ops to complete then + // in situation if host submits N_exe > QUEUE_SIZE, we lock here + // indefinitely + // ex: a_qrun(...) x QUEUE_SIZE + 1 (and no wait_qrun...) + // => deadlock + // 2. alternative would be wait to set exec_queue_full flag + // and wait until it is released + // 3. we can just exit and fail! -- most reasonable: + executor_handles[tmp_exec->id] = tmp_exec; + insert_op(OP, executor_handles[tmp_exec->id]); + ++num_active_executors; + break; + } + case CQ_CTRL_RUN_PQKERNEL: { + // Same logic as in CQ_CTRL_RUN_QKERNEL + if (num_active_executors >= __CQ_DEVICE_QUEUE_SIZE__) { + cq_log( + "%s [dispatch][RUN_PQKERNEL]: You have oversubsribed the executor " + "queue. We allow up to %d " + "concurrent executors per device. Exiting", + get_comm_source(), __CQ_DEVICE_QUEUE_SIZE__); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_RUNTIME_ERROR); + } + const int host_rank = CQ_MPI_HOST_RANK; + cq_exec * tmp_exec = NULL; + recv_exec_params(&tmp_exec, host_rank); + executor_handles[tmp_exec->id] = tmp_exec; + insert_op(OP, executor_handles[tmp_exec->id]); + ++num_active_executors; + break; + } + case CQ_CTRL_WAIT_EXEC: { + const int host_rank = CQ_MPI_HOST_RANK; + const size_t executor_id = recv_exec_id(host_rank); + if (executor_handles[executor_id] == NULL) { + cq_log( + "%s [dispatch][WAIT_EXEC]: device ehp is NULL. Returning. " + "Exiting\n", + get_comm_source()); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_RUNTIME_ERROR); + } + // NOTE: This part needs further tests to ensure no deadlock + device_wait_all_ops(); + comms_exec_wait(executor_handles[executor_id]); + send_exec_params(executor_handles[executor_id], host_rank); + // NOTE: This can be done when allocating new in offload + // i.e. clear old one and allocate new one + device_free_exec(&executor_handles[executor_id]); + --num_active_executors; + if (num_active_executors < 0) { + cq_log( + "%s [dispatch][WAIT_EXEC]: The number of active executors is < 0! " + "Should not happen. Exiting", + get_comm_source()); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_RUNTIME_ERROR); + } + break; + } + case CQ_CTRL_SYNC_EXEC: { + const int host_rank = CQ_MPI_HOST_RANK; + const size_t executor_id = recv_exec_id(host_rank); + if (executor_handles[executor_id] == NULL) { + cq_log( + "%s [dispatch][SYNC_EXEC]: device ehp is NULL. Returning. " + "Exiting\n", + get_comm_source()); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_RUNTIME_ERROR); + } + comms_exec_sync(executor_handles[executor_id]); + cq_log("%s [dispatch][SYNC_EXEC]: executor synced\n", get_comm_source()); + send_exec_params(executor_handles[executor_id], host_rank); + cq_log("%s [dispatch][SYNC_EXEC]: sent executor\n", get_comm_source()); + break; + } + case CQ_CTRL_WAIT: { + // wait for worker and send info to the host + // (which is blocked and waiting). + cq_log("%s [dispatch][WAIT]: Started waiting\n", get_comm_source()); + size_t num_ops = device_wait_all_ops(); + cq_log("%s [dispatch][WAIT]: Finished waiting\n", get_comm_source()); +#if CQ_CONF_QUEST_WITH_MPI + if (mpi_env.subcomm_rank == CQ_MPI_DEVICE_MASTER_RANK) { +#endif + const int host_rank = CQ_MPI_HOST_RANK; + cq_log("%s [dispatch]: sending num ops: %zu...\n", get_comm_source(), + num_ops); + MPI_Ssend(&num_ops, 1, MPI_UINT64_T, host_rank, CQ_MPI_WORLD_COMMS_TAG, + CQ_MPI_COMM_WORLD); + cq_log("%s [dispatch]: sent num ops.\n", get_comm_source()); +#if CQ_CONF_QUEST_WITH_MPI + } +#endif + break; + } + case CQ_CTRL_ABORT: { + const int host_rank = CQ_MPI_HOST_RANK; + const size_t executor_id = recv_exec_id(host_rank); + if (executor_handles[executor_id] == NULL) { + cq_log( + "%s [dispatch][ABORT]: device ehp is NULL. Returning. " + "Exiting\n", + get_comm_source()); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_RUNTIME_ERROR); + } + comms_exec_halt(executor_handles[executor_id]); + cq_log("%s [dispatch][ABORT]: executor halted\n", get_comm_source()); + break; + } + case CQ_CTRL_IDLE: { + break; + } + case CQ_CTRL_TEST: { + cq_log("%s [dispatch][TEST]: called test op\n", get_comm_source()); + break; + } + default: { + break; + } + } +} + +void device_wait_comms(void) { + pthread_mutex_lock(&dev_comm.comm_lock); + while (dev_comm.comm_busy) { + pthread_cond_wait(&dev_comm.cond_comm_busy, &dev_comm.comm_lock); + } + pthread_mutex_unlock(&dev_comm.comm_lock); +} + +// ---------------------------------------------------------------------------- +// Device Control Paramaters Comms +// ---------------------------------------------------------------------------- + +void host_comm_params(const enum ctrl_code OP, void * params) { + const int device_rank = CQ_MPI_DEVICE_RANK; + switch (OP) { + case CQ_CTRL_INIT: { + break; + } + case CQ_CTRL_FINALISE: { + break; + } + case CQ_CTRL_ALLOC: { + send_alloc_params(params, device_rank); + recv_alloc_params(params, device_rank); + break; + } + case CQ_CTRL_DEALLOC: { + send_alloc_params(params, device_rank); + recv_alloc_params(params, device_rank); + break; + } + case CQ_CTRL_RUN_QKERNEL: { + send_exec_params(params, device_rank); + break; + } + case CQ_CTRL_RUN_PQKERNEL: { + send_exec_params(params, device_rank); + break; + } + case CQ_CTRL_WAIT_EXEC: { + cq_exec * exec_params = (cq_exec *)params; + send_exec_id(((cq_exec *)params)->id, device_rank); + recv_exec_params(&exec_params, device_rank); + break; + } + case CQ_CTRL_SYNC_EXEC: { + cq_exec * exec_params = (cq_exec *)params; + send_exec_id(((cq_exec *)params)->id, device_rank); + recv_exec_params(&exec_params, device_rank); + break; + } + case CQ_CTRL_ABORT: { + send_exec_id(((cq_exec *)params)->id, device_rank); + break; + } + default: { + break; + } + } +} + +void recv_alloc_params(device_alloc_params * params, const int src) { + cq_log("%s [recv_alloc_params]: receiving...\n", get_comm_source()); + + if (!cq_validate_mpi_rank(src)) { + cq_log("%s [recv_exec_params]: encountered input error\n", + get_comm_source()); + return; + } + + const size_t params_size = sizeof(device_alloc_params); + MPI_Status status; + +#if CQ_CONF_QUEST_WITH_MPI + // device master (rank 0) gets from host from world comm + if (mpi_env.subcomm_rank == CQ_MPI_DEVICE_MASTER_RANK + || mpi_env.rank == CQ_MPI_HOST_RANK) { +#endif + + MPI_Recv(params, params_size, MPI_BYTE, src, CQ_MPI_WORLD_COMMS_TAG, + CQ_MPI_COMM_WORLD, &status); + +#if CQ_CONF_QUEST_WITH_MPI + } + // device-rank 0 brodcast params to rest of q-workers + if (mpi_env.rank != CQ_MPI_HOST_RANK) { + MPI_Bcast(params, params_size, MPI_BYTE, CQ_MPI_DEVICE_MASTER_RANK, + CQ_MPI_SPLIT_COMM); + } +#endif + + cq_log("%s [recv_alloc_params]: received.\n", get_comm_source()); + print_alloc_params(params); +} + +void send_alloc_params(const device_alloc_params * params, const int dest) { + if (params == NULL) { + cq_log("%s [recv_exec_params]: passed null params\n", get_comm_source()); + return; + } + + if (!cq_validate_mpi_rank(dest)) { + cq_log("%s [recv_exec_params]: encountered input error\n", + get_comm_source()); + return; + } + +#if CQ_CONF_QUEST_WITH_MPI + // device master (rank 0) gets from host from world comm + if (mpi_env.subcomm_rank == CQ_MPI_DEVICE_MASTER_RANK + || mpi_env.rank == CQ_MPI_HOST_RANK) { +#endif + + cq_log("%s [send_alloc_params]: sending...\n", get_comm_source()); + const size_t params_size = sizeof(device_alloc_params); + print_alloc_params(params); + MPI_Ssend(params, params_size, MPI_BYTE, dest, CQ_MPI_WORLD_COMMS_TAG, + CQ_MPI_COMM_WORLD); + cq_log("%s [send_alloc_params]: sent.\n", get_comm_source()); + +#if CQ_CONF_QUEST_WITH_MPI + } +#endif +} + +size_t recv_exec_id(const int src) { + cq_log("%s [recv_exec_id]: receiving...\n", get_comm_source()); + if (!cq_validate_mpi_rank(src)) { + cq_log("%s [recv_exec_params]: encountered input error\n", + get_comm_source()); + return -1; + } + + size_t id = -1; + MPI_Status status; + +#if CQ_CONF_QUEST_WITH_MPI + // device master (rank 0) gets from host from world comm + if (mpi_env.subcomm_rank == CQ_MPI_DEVICE_MASTER_RANK + || mpi_env.rank == CQ_MPI_HOST_RANK) { +#endif + + MPI_Recv(&id, 1, MPI_UINT64_T, src, CQ_MPI_WORLD_COMMS_TAG, + CQ_MPI_COMM_WORLD, &status); + +#if CQ_CONF_QUEST_WITH_MPI + } + // device-rank 0 brodcast params to rest of q-workers + if (mpi_env.rank != CQ_MPI_HOST_RANK) { + MPI_Bcast(&id, 1, MPI_UINT64_T, CQ_MPI_DEVICE_MASTER_RANK, + CQ_MPI_SPLIT_COMM); + } +#endif + + cq_log("%s [recv_exec_id]: received id: %zu\n", get_comm_source(), id); + return id; +} + +void send_exec_id(const size_t id, const int dest) { + if (!cq_validate_mpi_rank(dest)) { + cq_log("%s [recv_exec_params]: encountered input error\n", + get_comm_source()); + return; + } + +#if CQ_CONF_QUEST_WITH_MPI + // device master (rank 0) gets from host from world comm + if (mpi_env.subcomm_rank == CQ_MPI_DEVICE_MASTER_RANK + || mpi_env.rank == CQ_MPI_HOST_RANK) { +#endif + + cq_log("%s [send_exec_id]: sending id: %zu...\n", get_comm_source(), id); + MPI_Ssend(&id, 1, MPI_UINT64_T, dest, CQ_MPI_WORLD_COMMS_TAG, + CQ_MPI_COMM_WORLD); + cq_log("%s [send_exec_id]: sent\n", get_comm_source()); + +#if CQ_CONF_QUEST_WITH_MPI + } +#endif +} + +void recv_exec_params(cq_exec ** ehp, const int src) { + cq_log("%s [recv_exec_params]: receiving...\n", get_comm_source()); + + if (ehp == NULL) { + cq_log("%s [recv_exec_params]: passed null ehp\n", get_comm_source()); + return; + } + + if (!cq_validate_mpi_rank(src)) { + cq_log("%s [recv_exec_params]: encountered input error\n", + get_comm_source()); + return; + } + + int msg_size; + MPI_Status status; + +#if CQ_CONF_QUEST_WITH_MPI + // device-master (rank 0) gets from host from world comm + // or host gets from device-master + if (mpi_env.subcomm_rank == CQ_MPI_DEVICE_MASTER_RANK + || mpi_env.rank == CQ_MPI_HOST_RANK) { +#endif + + MPI_Recv(&msg_size, 1, MPI_INT, src, CQ_MPI_WORLD_COMMS_TAG, + CQ_MPI_COMM_WORLD, &status); + +#if CQ_CONF_QUEST_WITH_MPI + } + // device-rank 0 brodcast params to rest of q-workers + if (mpi_env.rank != CQ_MPI_HOST_RANK) { + MPI_Barrier(CQ_MPI_SPLIT_COMM); + MPI_Bcast(&msg_size, 1, MPI_INT, CQ_MPI_DEVICE_MASTER_RANK, + CQ_MPI_SPLIT_COMM); + } +#endif + + void * recv_buffer = malloc(msg_size); + + // reserve space for all the data + currently unused members + if (*ehp == NULL) { + cq_log("%s [recv_exec_params]: *ehp is NULL. Allocating on device.\n", + get_comm_source()); + + *ehp = (cq_exec *)malloc(sizeof(cq_exec)); + pthread_mutex_init(&(*ehp)->lock, NULL); + pthread_cond_init(&(*ehp)->cond_exec_complete, NULL); + } else { + cq_log( + "%s [recv_exec_params]: *ehp is already allocated. So I'm on " + "host.\n", + get_comm_source()); + } + pthread_mutex_lock(&(*ehp)->lock); + + if (recv_buffer == NULL || *ehp == NULL) { + cq_log("%s [recv_exec_params]: malloc failed. Exiting\n", + get_comm_source()); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_MALLOC_ERROR); + } + +#if CQ_CONF_QUEST_WITH_MPI + // device-master (rank 0) gets from host from world comm + // or host gets from device-master + if (mpi_env.subcomm_rank == CQ_MPI_DEVICE_MASTER_RANK + || mpi_env.rank == CQ_MPI_HOST_RANK) { +#endif + + MPI_Recv(recv_buffer, msg_size, MPI_PACKED, src, CQ_MPI_WORLD_COMMS_TAG, + CQ_MPI_COMM_WORLD, &status); + +#if CQ_CONF_QUEST_WITH_MPI + } + // device-rank 0 brodcast params to rest of q-workers + if (mpi_env.rank != CQ_MPI_HOST_RANK) { + MPI_Barrier(CQ_MPI_SPLIT_COMM); + MPI_Bcast(recv_buffer, msg_size, MPI_PACKED, CQ_MPI_DEVICE_MASTER_RANK, + CQ_MPI_SPLIT_COMM); + } +#endif + + const size_t bool_size = sizeof(bool); + const size_t cq_status_size = sizeof(cq_status); + // unpack + int position = 0; + MPI_Unpack(recv_buffer, msg_size, &position, &(*ehp)->id, 1, MPI_UINT64_T, + CQ_MPI_COMM_WORLD); + MPI_Unpack(recv_buffer, msg_size, &position, &(*ehp)->exec_init, bool_size, + MPI_BYTE, CQ_MPI_COMM_WORLD); + MPI_Unpack(recv_buffer, msg_size, &position, &(*ehp)->complete, bool_size, + MPI_BYTE, CQ_MPI_COMM_WORLD); + MPI_Unpack(recv_buffer, msg_size, &position, &(*ehp)->halt, bool_size, + MPI_BYTE, CQ_MPI_COMM_WORLD); + MPI_Unpack(recv_buffer, msg_size, &position, &(*ehp)->status, cq_status_size, + MPI_BYTE, CQ_MPI_COMM_WORLD); + MPI_Unpack(recv_buffer, msg_size, &position, &(*ehp)->nqubits, 1, + MPI_UINT64_T, CQ_MPI_COMM_WORLD); + MPI_Unpack(recv_buffer, msg_size, &position, &(*ehp)->completed_shots, 1, + MPI_UINT64_T, CQ_MPI_COMM_WORLD); + MPI_Unpack(recv_buffer, msg_size, &position, &(*ehp)->expected_shots, 1, + MPI_UINT64_T, CQ_MPI_COMM_WORLD); + MPI_Unpack(recv_buffer, msg_size, &position, &(*ehp)->nmeasure, 1, + MPI_UINT64_T, CQ_MPI_COMM_WORLD); + MPI_Unpack(recv_buffer, msg_size, &position, &(*ehp)->params_size, 1, + MPI_UINT64_T, CQ_MPI_COMM_WORLD); + + size_t fname_size = 0; + MPI_Unpack(recv_buffer, msg_size, &position, &fname_size, 1, MPI_UINT64_T, + CQ_MPI_COMM_WORLD); + fname_size *= sizeof(char); + + const size_t qreg_size = sizeof(qubit) * (*ehp)->nqubits; + const size_t num_shots = (*ehp)->expected_shots; + const size_t creg_size = sizeof(cstate) * (*ehp)->nmeasure * num_shots; + const size_t params_size = (*ehp)->params_size; + + // when on host the resources are already allocated! + if (mpi_env.rank != CQ_MPI_HOST_RANK) { + (*ehp)->fname = (char *)malloc(fname_size); + if ((*ehp)->fname == NULL) { + cq_log("%s [recv_exec_params]: malloc *ehp->fname failed. Exiting\n", + get_comm_source()); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_MALLOC_ERROR); + } + + (*ehp)->qreg = (qubit *)malloc(qreg_size); + if ((*ehp)->qreg == NULL) { + cq_log("%s [recv_exec_params]: malloc *ehp->qreg failed. Exiting\n", + get_comm_source()); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_MALLOC_ERROR); + } + + (*ehp)->creg = (cstate *)malloc(creg_size); + if ((*ehp)->creg == NULL) { + cq_log("%s [recv_exec_params]: malloc *ehp->creg failed. Exiting\n", + get_comm_source()); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_MALLOC_ERROR); + } + + (*ehp)->params = malloc(params_size); + if ((*ehp)->params == NULL) { + cq_log("%s [recv_exec_params]: malloc *ehp->params failed. Exiting\n", + get_comm_source()); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_MALLOC_ERROR); + } + } + + MPI_Unpack(recv_buffer, msg_size, &position, (*ehp)->fname, fname_size, + MPI_CHAR, CQ_MPI_COMM_WORLD); + MPI_Unpack(recv_buffer, msg_size, &position, (char *)(*ehp)->qreg, qreg_size, + MPI_BYTE, CQ_MPI_COMM_WORLD); + MPI_Unpack(recv_buffer, msg_size, &position, (char *)(*ehp)->creg, creg_size, + MPI_BYTE, CQ_MPI_COMM_WORLD); + MPI_Unpack(recv_buffer, msg_size, &position, (char *)(*ehp)->params, + params_size, MPI_BYTE, CQ_MPI_COMM_WORLD); + + free(recv_buffer); + + cq_log("%s [recv_exec_params]: received.\n", get_comm_source()); + print_ehp(*ehp); + pthread_mutex_unlock(&(*ehp)->lock); +} + +void send_exec_params(cq_exec * ehp, const int dest) { + if (ehp == NULL) { + cq_log("%s [recv_exec_params]: passed null ehp\n", get_comm_source()); + return; + } + + if (!cq_validate_mpi_rank(dest)) { + cq_log("%s [recv_exec_params]: encountered input error\n", + get_comm_source()); + return; + } + +// if running with MPI QuEST, we don't need to communicate +// with quantum workers because the results from QuEST +// (e.g. measurements) should be already synchronised. +// so only device-master communicates its state to host. +#if CQ_CONF_QUEST_WITH_MPI + // device master (rank 0) gets from host from world comm + if (mpi_env.subcomm_rank == CQ_MPI_DEVICE_MASTER_RANK + || mpi_env.rank == CQ_MPI_HOST_RANK) { +#endif + + pthread_mutex_lock(&ehp->lock); + cq_log("%s [send_exec_params]: sending...\n", get_comm_source()); + print_ehp(ehp); + + const size_t bool_size = sizeof(bool); + const size_t cq_status_size = sizeof(cq_status); + int max_buffer_size = 0; + int member_size = 0; + MPI_Pack_size(1, MPI_UINT64_T, CQ_MPI_COMM_WORLD, + &max_buffer_size); // id + MPI_Pack_size(bool_size, MPI_BYTE, CQ_MPI_COMM_WORLD, + &member_size); // exec_init + max_buffer_size += member_size; + MPI_Pack_size(bool_size, MPI_BYTE, CQ_MPI_COMM_WORLD, + &member_size); // complete + max_buffer_size += member_size; + MPI_Pack_size(bool_size, MPI_BYTE, CQ_MPI_COMM_WORLD, + &member_size); // halt + max_buffer_size += member_size; + MPI_Pack_size(cq_status_size, MPI_BYTE, CQ_MPI_COMM_WORLD, + &member_size); // status + max_buffer_size += member_size; + MPI_Pack_size(1, MPI_UINT64_T, CQ_MPI_COMM_WORLD, + &member_size); // nqubits + max_buffer_size += member_size; + MPI_Pack_size(1, MPI_UINT64_T, CQ_MPI_COMM_WORLD, + &member_size); // completed_shots + max_buffer_size += member_size; + MPI_Pack_size(1, MPI_UINT64_T, CQ_MPI_COMM_WORLD, + &member_size); // expected_shots + max_buffer_size += member_size; + MPI_Pack_size(1, MPI_UINT64_T, CQ_MPI_COMM_WORLD, + &member_size); // nmeasure + max_buffer_size += member_size; + MPI_Pack_size(1, MPI_UINT64_T, CQ_MPI_COMM_WORLD, &member_size); + max_buffer_size += member_size; // params_size + + size_t fname_size = 0; + if (ehp->fname != NULL) { + fname_size = strlen(ehp->fname) + 1; + } + MPI_Pack_size(1, MPI_UINT64_T, CQ_MPI_COMM_WORLD, + &member_size); // fname_size + max_buffer_size += member_size; + MPI_Pack_size(fname_size, MPI_CHAR, CQ_MPI_COMM_WORLD, + &member_size); // fname + max_buffer_size += member_size; + + const size_t qreg_size = sizeof(qubit) * ehp->nqubits; + MPI_Pack_size(qreg_size, MPI_BYTE, CQ_MPI_COMM_WORLD, + &member_size); // qreg + max_buffer_size += member_size; + + const size_t num_shots = ehp->expected_shots; + const size_t creg_size = sizeof(cstate) * ehp->nmeasure * num_shots; + + MPI_Pack_size(creg_size, MPI_BYTE, CQ_MPI_COMM_WORLD, + &member_size); // creg + max_buffer_size += member_size; + + const size_t params_size = ehp->params_size; + MPI_Pack_size(params_size, MPI_BYTE, CQ_MPI_COMM_WORLD, &member_size); + max_buffer_size += member_size; // params + + void * send_buffer = malloc(max_buffer_size); + if (send_buffer == NULL) { + cq_log( + "Failed to allocate buffer for sending executor handle. Exiting\n"); + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_MALLOC_ERROR); + } + int position = 0; + MPI_Pack(&ehp->id, 1, MPI_UINT64_T, send_buffer, max_buffer_size, &position, + CQ_MPI_COMM_WORLD); + MPI_Pack(&ehp->exec_init, bool_size, MPI_BYTE, send_buffer, max_buffer_size, + &position, CQ_MPI_COMM_WORLD); + MPI_Pack(&ehp->complete, bool_size, MPI_BYTE, send_buffer, max_buffer_size, + &position, CQ_MPI_COMM_WORLD); + MPI_Pack(&ehp->halt, bool_size, MPI_BYTE, send_buffer, max_buffer_size, + &position, CQ_MPI_COMM_WORLD); + MPI_Pack(&ehp->status, cq_status_size, MPI_BYTE, send_buffer, + max_buffer_size, &position, CQ_MPI_COMM_WORLD); + + MPI_Pack(&ehp->nqubits, 1, MPI_UINT64_T, send_buffer, max_buffer_size, + &position, CQ_MPI_COMM_WORLD); + MPI_Pack(&ehp->completed_shots, 1, MPI_UINT64_T, send_buffer, + max_buffer_size, &position, CQ_MPI_COMM_WORLD); + MPI_Pack(&ehp->expected_shots, 1, MPI_UINT64_T, send_buffer, + max_buffer_size, &position, CQ_MPI_COMM_WORLD); + MPI_Pack(&ehp->nmeasure, 1, MPI_UINT64_T, send_buffer, max_buffer_size, + &position, CQ_MPI_COMM_WORLD); + MPI_Pack(&ehp->params_size, 1, MPI_UINT64_T, send_buffer, max_buffer_size, + &position, CQ_MPI_COMM_WORLD); + + // NOTE: skip pthread stuff... + + MPI_Pack(&fname_size, 1, MPI_UINT64_T, send_buffer, max_buffer_size, + &position, CQ_MPI_COMM_WORLD); + MPI_Pack(ehp->fname, fname_size, MPI_CHAR, send_buffer, max_buffer_size, + &position, CQ_MPI_COMM_WORLD); + + MPI_Pack(ehp->qreg, qreg_size, MPI_BYTE, send_buffer, max_buffer_size, + &position, CQ_MPI_COMM_WORLD); + MPI_Pack(ehp->creg, creg_size, MPI_BYTE, send_buffer, max_buffer_size, + &position, CQ_MPI_COMM_WORLD); + MPI_Pack(ehp->params, params_size, MPI_BYTE, send_buffer, max_buffer_size, + &position, CQ_MPI_COMM_WORLD); + + MPI_Ssend(&position, 1, MPI_INT, dest, CQ_MPI_WORLD_COMMS_TAG, + CQ_MPI_COMM_WORLD); + + MPI_Ssend(send_buffer, position, MPI_PACKED, dest, CQ_MPI_WORLD_COMMS_TAG, + CQ_MPI_COMM_WORLD); + + free(send_buffer); + cq_log("%s [send_exec_params]: sent.\n", get_comm_source()); + pthread_mutex_unlock(&ehp->lock); +#if CQ_CONF_QUEST_WITH_MPI + } +#endif +} + +void device_free_exec(cq_exec ** ehp) { + cq_log("%s [device_free_exec]: freeing the executor\n", get_comm_source()); + if (ehp == NULL) { + cq_log("%s [device_free_exec]: ehp is NULL\n", get_comm_source()); + return; + } + + if (*ehp == NULL) { + cq_log("%s [device_free_exec]: *ehp is NULL\n", get_comm_source()); + return; + } + + pthread_mutex_lock(&(*ehp)->lock); + if ((*ehp)->fname != NULL) { + free((*ehp)->fname); + (*ehp)->fname = NULL; + } + + if ((*ehp)->qreg != NULL) { + free((*ehp)->qreg); + (*ehp)->qreg = NULL; + } + + if ((*ehp)->creg != NULL) { + free((*ehp)->creg); + (*ehp)->creg = NULL; + } + + if ((*ehp)->params != NULL) { + free((*ehp)->params); + (*ehp)->params = NULL; + } + + pthread_mutex_unlock(&(*ehp)->lock); + pthread_mutex_destroy(&(*ehp)->lock); + pthread_cond_destroy(&(*ehp)->cond_exec_complete); + if ((*ehp) != NULL) { + free((*ehp)); + (*ehp) = NULL; + } + + cq_log("%s [device_free_exec]: freed the executor\n", get_comm_source()); +} + +// ---------------------------------------------------------------------------- +// Helpers +// ---------------------------------------------------------------------------- + +const char * get_comm_source(void) { + if (mpi_env.rank == CQ_MPI_HOST_RANK) { + return "Host:\t\t\t\t"; + } else if (mpi_env.rank == CQ_MPI_DEVICE_RANK) { + return "Device:\t\t\t\t"; + } else if (is_quantum_worker()) { + return debug_worker_name; + } else { + return ""; + } +} + +const char * op_to_str(const enum ctrl_code OP) { + switch (OP) { + case CQ_CTRL_IDLE: { + return "CQ_CTRL_IDLE"; + break; + } + case CQ_CTRL_ALLOC: { + return "CQ_CTRL_ALLOC"; + break; + } + case CQ_CTRL_DEALLOC: { + return "CQ_CTRL_DEALLOC"; + break; + } + case CQ_CTRL_INIT: { + return "CQ_CTRL_INIT"; + break; + } + case CQ_CTRL_FINALISE: { + return "CQ_CTRL_FINALISE"; + break; + } + case CQ_CTRL_WAIT: { + return "CQ_CTRL_WAIT"; + break; + } + case CQ_CTRL_RUN_QKERNEL: { + return "CQ_CTRL_RUN_QKERNEL"; + break; + } + case CQ_CTRL_RUN_PQKERNEL: { + return "CQ_CTRL_RUN_PQKERNEL"; + break; + } + case CQ_CTRL_TEST: { + return "CQ_CTRL_TEST"; + break; + } + case CQ_CTRL_ABORT: { + return "CQ_CTRL_ABORT"; + break; + } + case CQ_CTRL_WAIT_EXEC: { + return "CQ_CTRL_WAIT_EXEC"; + break; + } + case CQ_CTRL_SYNC_EXEC: { + return "CQ_CTRL_SYNC_EXEC"; + break; + } + default: { + break; + } + } + return ""; +} + +void print_alloc_params(const device_alloc_params * params) { + if (params == NULL) { + return; + } + cq_log("%s alloc params: NQUBITS: %zu, qreg_idx: %zu, STATUS: %d\n", + get_comm_source(), params->NQUBITS, params->qregistry_idx, + params->status); +} + +void print_ehp(const cq_exec * ehp) { + if (ehp == NULL) { + cq_log("ehp is NULL\n"); + } + + cq_log( + "%s ehp details:\nid: %zu, exec_init: %d, complete: %d, halt: %d, " + "STATUS: " + "%d\nNQUBITS: " + "%zu, completed_shots: %zu, expected_shots: %zu, NMEASURE: " + "%zu, params_size: %zu\nfname: " + "%s\nqreg:\n", + get_comm_source(), ehp->id, ehp->exec_init, ehp->complete, ehp->halt, + ehp->status, ehp->nqubits, ehp->completed_shots, ehp->expected_shots, + ehp->nmeasure, ehp->params_size, ehp->fname); + + for (size_t i = 0; i < ehp->nqubits; ++i) { + cq_log("qubit[%zu]: reg_idx: %zu, offset: %zu, N: %zu\n", i, + ehp->qreg[i].registry_index, ehp->qreg[i].offset, ehp->qreg[i].N); + } + cq_log("\ncreg:\n"); + + // for (size_t i = 0; i < ehp->nmeasure * ehp->expected_shots; ++i) { + // cq_log("cstate[%zu]: %d\n", i, ehp->creg[i]); + // } + + cq_log("\nparams:\n"); + double * dparams = (double *)ehp->params; + for (size_t i = 0; i < ehp->params_size / sizeof(double); ++i) { + cq_log("params[%zu]: %f\n", i, dparams[i]); + } + cq_log("%s ehp details END\n\n", get_comm_source()); +} + +bool is_quantum_worker(void) { + return mpi_env.rank > 0; +} + +void validate_nproc(const int nproc) { + const int device_nproc = nproc - 1; + if (nproc == 1) { + cq_log("Initialising CQ MPI with 1 process. Running serial job.\n"); + CQ_MPI_DEVICE_RANK = CQ_MPI_HOST_RANK; + return; + } +#if CQ_CONF_QUEST_WITH_MPI + if (!((device_nproc > 0) && ((device_nproc & (device_nproc - 1)) == 0))) { + if (mpi_env.rank == CQ_MPI_HOST_RANK) { + cq_log( + "Incorrect number of MPI processes. The (N - 1) should be power of " + "2!\n"); + } + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_RUNTIME_ERROR); + } +#endif +#ifndef CQ_CONF_QUEST_WITH_MPI + if (device_nproc != 1) { + if (mpi_env.rank == CQ_MPI_HOST_RANK) { + cq_log( + "Incorrect number of MPI processes. If QuEST uses multi-threading " + "only, there should be only 2 MPI processes used for CQ!\n"); + } + MPI_Abort(CQ_MPI_COMM_WORLD, CQ_MPI_RUNTIME_ERROR); + } +#endif + // default setup + CQ_MPI_DEVICE_RANK = CQ_MPI_HOST_RANK + 1; +} + +// #undef CQ_MPI_HOST_RANK +// #undef CQ_MPI_DEVICE_RANK +// #undef CQ_MPI_DEVICE_MASTER_RANK +// #undef CQ_MPI_WORLD_COMMS_TAG +// #undef CQ_MPI_SUBCOMMS_TAG +// #undef CQ_MPI_RUNTIME_ERROR +// #undef CQ_MPI_MALLOC_ERROR diff --git a/src/host-device/comms/comms_mpi.h b/src/host-device/comms/comms_mpi.h new file mode 100644 index 0000000..05309e6 --- /dev/null +++ b/src/host-device/comms/comms_mpi.h @@ -0,0 +1,154 @@ +#ifndef CQ_HOST_DEVICE_COMMS_MPI_H +#define CQ_HOST_DEVICE_COMMS_MPI_H + +#include "datatypes.h" +#include "src/host-device/comms.h" +#include "src/host/opcodes.h" + +#include + +#include +#include + +/// +/// initialises MPI environment both on host and on the device. +/// @param VERBOSITY unsigned integer controlling diagnostic output. +void init_host_device_mpi(const unsigned int VERBOSITY); + +/// +/// finalises MPI environment both on host and on the device. +/// @param VERBOSITY unsigned integer controlling diagnostic output. +void finalise_host_device_mpi(const unsigned int VERBOSITY); + +// ---------------------------------------------------------------------------- +// Device Comm Ops +// ---------------------------------------------------------------------------- + +/// +/// initialises on-device communication thread responsible for MPI messaging +/// with host. +/// @param VERBOSITY unsigned integer controlling diagnostic output. +void device_init_comms(const unsigned int VERBOSITY); + +/// +/// finalises on-device communication thread responsible for MPI messaging +/// with host. +/// @param VERBOSITY unsigned integer controlling diagnostic output. +void device_finalise_comms(const unsigned int VERBOSITY); + +/// +/// starts listening for the incoming messages from the host. +/// @param args arbitrary arguments to function to satisfy pthread function +/// signature +/// @return arbitrary return data to satisfy pthread function signature. +/// Currently always returns NULL. +void * device_listen(void * args); + +/// +/// dispatches recieved control operation to the worker thread. +/// @param OP an enum argument specifying CQ_CTRL_OP +void device_dispatch_ctrl_op(const enum ctrl_code OP); + +/// +/// blocks device master thread and awaits for the comms to complete. +/// it needs to be called only once at the end to ensure that the main thread +/// does not clean-up and close MPI before work is done on the worker. +void device_wait_comms(void); + +// ---------------------------------------------------------------------------- +// Device Control Paramaters Comms +// ---------------------------------------------------------------------------- + +/// +/// communicates control parameters between host and device. +/// @param OP an enum argument specifying CQ_CTRL_OP +/// @param[in,out] params void pointer to arbitrary params +void host_comm_params(const enum ctrl_code OP, void * params); + +/// +/// receives allocation parameters from the source. +/// @param[out] params reference to parameters to store the results of +/// communication +/// @param src source rank of incoming message +void recv_alloc_params(device_alloc_params * params, const int src); + +/// +/// sends allocation parameters to the destination. +/// @param[in] params reference to parameters to communicate +/// @param dest destination rank of outgoing message +void send_alloc_params(const device_alloc_params * params, const int dest); + +/// +/// receives executor id used for matching results with correct executor. +/// @param src source rank of incoming message +/// @return executor id +size_t recv_exec_id(const int src); + +/// +/// sends executor id used for matching results with correct executor. +/// @param id executor id to be sent +/// @param dest destination rank of outgoing message +void send_exec_id(const size_t id, const int dest); + +/// +/// receives executor handle from the source. Also, if called on device, +/// allocates executor in on-device memory, which then needs to be freed using +/// device_free_exec. +/// @param[out] ehp executor handle used for host-device offloading +/// @param src source rank of incoming message +void recv_exec_params(cq_exec ** ehp, const int src); + +/// sends updated executor handle to the destination. +/// @param[in] ehp executor handle used for host-device offloading +/// @param dest destination rank of outgoing message +void send_exec_params(cq_exec * ehp, const int dest); + +/// +/// frees on-device memory pointed to by executor handle. +/// @param[in,out] ehp executor handle used for host-device offloading +void device_free_exec(cq_exec ** ehp); + +// ---------------------------------------------------------------------------- +// Helpers +// ---------------------------------------------------------------------------- + +/// +/// returns the name of either host or the device. Used for printing +/// diagnostics. +/// @return name of the device (or host). +const char * get_comm_source(void); + +/// +/// returns the current MPI rank. +/// @return MPI rank of a given process. +int get_rank(void); + +/// +/// converts CQ control code to string. +/// @param OP an enum argument specifying CQ_CTRL_OP +/// @return string represntation of OP. +const char * op_to_str(const enum ctrl_code OP); + +/// +/// prints values of the alloc parameters. +/// @param[in] params reference to parameters +void print_alloc_params(const device_alloc_params * params); + +/// +/// prints members of the executor. +/// @param[in] ehp reference to executor +void print_ehp(const cq_exec * ehp); + +// +/// +/// check if MPI process is a quantum worker (not resposible for host-device +/// communication). +/// @return +bool is_quantum_worker(void); + +/// validates the number of MPI processes to meet QuEST constraints. +/// Exits program if check failed. +/// @param nproc number of MPI processes +void validate_nproc(const int nproc); + +#endif diff --git a/src/host-device/kernel_utils.c b/src/host-device/kernel_utils.c index 5afc0a7..19b0c5a 100644 --- a/src/host-device/kernel_utils.c +++ b/src/host-device/kernel_utils.c @@ -4,6 +4,10 @@ #include #include "kernel_utils.h" +//#include "mpi_comms.h" +#include "src/host-device/comms.h" +#include + struct qkern_registry qk_reg; struct pqkern_registry pqk_reg; @@ -29,10 +33,39 @@ cq_status register_qkern(qkern kernel) { } } + //host_device_sync_comms(); + RUN_HOST_ONLY(); + host_wait_all_ops(); return status; } cq_status register_pqkern(pqkern kernel) { + cq_status status = CQ_ERROR; + char * fname; + + if (kernel != NULL && pqk_reg.next_available_slot < __CQ_MAX_NUM_QKERN__) { + status = find_pqkern_name(kernel, &fname); + if (status == CQ_SUCCESS) { + // This kernel has already been registered! + status = CQ_WARNING; + } else { + pqkern_map * pqkmap = &pqk_reg.pqkernels[pqk_reg.next_available_slot]; + kernel(0, NULL, 0, NULL, NULL, pqkmap); + if (pqkmap->fname[0] != '\0') { + pqkmap->fn = kernel; + ++pqk_reg.next_available_slot; + status = CQ_SUCCESS; + } else { + status = CQ_ERROR; + } + } + } + + //host_device_sync_comms(); + RUN_HOST_ONLY(); + host_wait_all_ops(); + return status; + return CQ_ERROR; } @@ -74,7 +107,7 @@ cq_status find_pqkern_pointer(char const * const FNAME, pqkern * pqk) { *pqk = NULL; int status = CQ_SUCCESS; - for (size_t i = 0; i < qk_reg.next_available_slot; ++i) { + for (size_t i = 0; i < pqk_reg.next_available_slot; ++i) { if (!strcmp(FNAME, pqk_reg.pqkernels[i].fname)) { // We found it! *pqk = pqk_reg.pqkernels[i].fn; @@ -91,7 +124,7 @@ cq_status find_pqkern_name(pqkern const PQK, char ** fname) { *fname = NULL; int status = CQ_SUCCESS; - for (size_t i = 0; i < qk_reg.next_available_slot; ++i) { + for (size_t i = 0; i < pqk_reg.next_available_slot; ++i) { if (PQK == pqk_reg.pqkernels[i].fn) { // We found it! *fname = pqk_reg.pqkernels[i].fname; @@ -105,6 +138,7 @@ cq_status find_pqkern_name(pqkern const PQK, char ** fname) { } void init_exec_handle(const size_t NQUBITS, const size_t NSHOTS, const size_t NMEASURE, cq_exec * ehp) { + ehp->id = assign_exec_id(); ehp->exec_init = true; ehp->complete = false; ehp->halt = false; @@ -113,6 +147,7 @@ void init_exec_handle(const size_t NQUBITS, const size_t NSHOTS, const size_t NM ehp->completed_shots = 0; ehp->expected_shots = NSHOTS; ehp->nmeasure = NMEASURE; + ehp->params_size = 0; ehp->fname = NULL; ehp->qreg = NULL; ehp->creg = NULL; diff --git a/src/host/env.c b/src/host/env.c index 187c6ab..4d08763 100644 --- a/src/host/env.c +++ b/src/host/env.c @@ -1,25 +1,26 @@ -#include -#include -#include +#include "env.h" + #include "datatypes.h" #include "src/host-device/comms.h" -#include "opcodes.h" -#include "env.h" -struct cq_environment cq_env = { - .initialised = false, - .finalised = false -}; +#include +#include +#include +#include + +struct cq_environment cq_env = { .initialised = false, .finalised = false }; struct dev_link dev_ctrl; cq_status cq_init(const unsigned int VERBOSITY) { - cq_status status = CQ_SUCCESS; + cq_status status = CQ_SUCCESS; if (!cq_env.finalised) { - if (!cq_env.initialised) { + if (!cq_env.initialised) { if (VERBOSITY > 0) { - printf("Initialising CQ Simulated Backend library. QuEST environment report to follow.\n\n"); + printf( + "Initialising CQ Simulated Backend library. QuEST environment " + "report to follow.\n\n"); } initialise_device(VERBOSITY); @@ -32,19 +33,54 @@ cq_status cq_init(const unsigned int VERBOSITY) { status = CQ_WARNING; } } else { - printf("CQ-SimBE cannot be reinitialised once finalised! This would break QuEST.\n"); + printf( + "CQ-SimBE cannot be reinitialised once finalised! This would break " + "QuEST.\n"); + status = CQ_ERROR; + } + + return status; +} + +#if CQ_WITH_MPI_COMMS +cq_status cq_init_custom_mpi_comm(MPI_Comm cq_comm, + const unsigned int VERBOSITY) { + cq_status status = CQ_SUCCESS; + + if (!cq_env.finalised) { + if (!cq_env.initialised) { + if (VERBOSITY > 0) { + printf( + "Initialising CQ Simulated Backend library. QuEST environment " + "report to follow.\n\n"); + } + + initialise_device_with_custom_mpi_comm(cq_comm, VERBOSITY); + + cq_env.initialised = true; + } else { + if (VERBOSITY > 0) { + printf("CQ-SimBE is already initialised. No need to do it again.\n"); + } + status = CQ_WARNING; + } + } else { + printf( + "CQ-SimBE cannot be reinitialised once finalised! This would break " + "QuEST.\n"); status = CQ_ERROR; } return status; } +#endif + cq_status cq_finalise(const unsigned int VERBOSITY) { cq_status status = CQ_SUCCESS; if (!cq_env.finalised) { - if (VERBOSITY > 0) - printf("Host finalising\n"); + if (VERBOSITY > 0) printf("Host finalising\n"); finalise_device(VERBOSITY); cq_env.finalised = true; @@ -57,3 +93,15 @@ cq_status cq_finalise(const unsigned int VERBOSITY) { return status; } + +// fortran helper -- just don't use it from C +cq_status fort_init_custom_mpi_comm(int * cq_comm, + const unsigned int VERBOSITY) { +#if CQ_WITH_MPI_COMMS + MPI_Comm c_cq_comm; + c_cq_comm = MPI_Comm_f2c(*cq_comm); + return cq_init_custom_mpi_comm(c_cq_comm, VERBOSITY); +#endif + + return cq_init(VERBOSITY); +} diff --git a/src/host/fortran/host_int.f90 b/src/host/fortran/host_int.f90 index 6abb607..d466892 100644 --- a/src/host/fortran/host_int.f90 +++ b/src/host/fortran/host_int.f90 @@ -13,6 +13,14 @@ function fortran_cq_init(VERBOSITY) bind(C, name="cq_init") integer(c_int) :: fortran_cq_init end function fortran_cq_init + function fortran_cq_init_custom_mpi_comm(cq_comm, VERBOSITY) bind(C, name="fort_init_custom_mpi_comm") + use, intrinsic :: iso_c_binding, only: c_int + implicit none + integer(c_int) :: cq_comm + integer(c_int), value :: VERBOSITY ! passes integer + integer(c_int) :: fortran_cq_init_custom_mpi_comm + end function fortran_cq_init_custom_mpi_comm + function fortran_cq_finalise(VERBOSITY) bind(C, name="cq_finalise") use, intrinsic :: iso_c_binding, only: c_int implicit none @@ -20,6 +28,11 @@ function fortran_cq_finalise(VERBOSITY) bind(C, name="cq_finalise") integer(c_int) :: fortran_cq_finalise end function fortran_cq_finalise + function is_device() bind(C, name="is_device") + use, intrinsic :: iso_c_binding, only: c_bool + logical(c_bool) :: is_device + end function is_device + !cq_status alloc_qubit(qubit ** qhp); function alloc_qubit(qhp) bind(C) use, intrinsic :: iso_c_binding, only: c_int, c_ptr diff --git a/src/host/fortran/host_ops.f90 b/src/host/fortran/host_ops.f90 index 5020735..c86c90e 100644 --- a/src/host/fortran/host_ops.f90 +++ b/src/host/fortran/host_ops.f90 @@ -11,10 +11,18 @@ status = fortran_cq_init(VERBOSITY) end procedure cq_init + module procedure cq_init_custom_mpi_comm !(cq_comm, VERBOSITY) result(status) + status = fortran_cq_init_custom_mpi_comm(cq_comm, VERBOSITY) + end procedure cq_init_custom_mpi_comm + module procedure cq_finalise !(VERBOSITY) result(status) status = fortran_cq_finalise(VERBOSITY) end procedure cq_finalise + module procedure cq_is_device !() result(res) + res = is_device() + end procedure cq_is_device + module procedure cq_alloc_qubit !(qhp) result(status) status = alloc_qubit(qhp%this) end procedure cq_alloc_qubit diff --git a/src/host/host_ops.c b/src/host/host_ops.c index 3cc98fb..2e34a65 100644 --- a/src/host/host_ops.c +++ b/src/host/host_ops.c @@ -1,9 +1,13 @@ -#include -#include "datatypes.h" #include "host_ops.h" + +#include "datatypes.h" #include "kernel_utils.h" #include "src/host-device/comms.h" +#include + +// #include "src/host-device/mpi_comms.h" +#include // Resource Management cq_status alloc_qubit(qubit ** qhp) { @@ -11,21 +15,22 @@ cq_status alloc_qubit(qubit ** qhp) { } cq_status alloc_qureg(qubit ** qrp, size_t N) { + RUN_HOST_ONLY(); cq_status status = CQ_SUCCESS; - + // check qr is NULL // If it's not, we still allocate, but issue a CQ_WARNING. - if (*qrp != NULL) { + if (*qrp != NULL) { status = CQ_WARNING; } - + device_alloc_params alloc_params = { .NQUBITS = N, .qregistry_idx = 0, - .status = CQ_ERROR + .status = CQ_ERROR, }; - *qrp = (qubit *) malloc(sizeof(qubit) * N); + *qrp = (qubit *)malloc(sizeof(qubit) * N); if (*qrp == NULL) { /* whoops, the malloc failed originally we only malloc'd on the host after receiving @@ -35,7 +40,7 @@ cq_status alloc_qureg(qubit ** qrp, size_t N) { */ return CQ_ERROR; } - + host_send_ctrl_op(CQ_CTRL_ALLOC, &alloc_params); host_wait_all_ops(); @@ -61,12 +66,13 @@ cq_status free_qubit(qubit ** qhp) { } cq_status free_qureg(qubit ** qrp) { + RUN_HOST_ONLY(); if (*qrp == NULL) return CQ_WARNING; device_alloc_params dealloc_params = { .NQUBITS = 0, .qregistry_idx = (*qrp)[0].registry_index, - .status = CQ_ERROR + .status = CQ_ERROR, }; host_send_ctrl_op(CQ_CTRL_DEALLOC, &dealloc_params); @@ -82,13 +88,17 @@ cq_status free_qureg(qubit ** qrp) { // Executors -cq_status s_qrun(qkern kernel, qubit * qrp, const size_t NQUBITS, -cstate * crp, const size_t NMEASURE) { +cq_status s_qrun(qkern kernel, + qubit * qrp, + const size_t NQUBITS, + cstate * crp, + const size_t NMEASURE) { + RUN_HOST_ONLY(); cq_status status = CQ_ERROR; cq_exec exec_handle; status = a_qrun(kernel, qrp, NQUBITS, crp, NMEASURE, &exec_handle); - + if (status == CQ_SUCCESS) { status = wait_qrun(&exec_handle); @@ -100,8 +110,13 @@ cstate * crp, const size_t NMEASURE) { return status; } -cq_status a_qrun(qkern kernel, qubit * qrp, const size_t NQUBITS, -cstate * crp, const size_t NMEASURE, cq_exec * const ehp) { +cq_status a_qrun(qkern kernel, + qubit * qrp, + const size_t NQUBITS, + cstate * crp, + const size_t NMEASURE, + cq_exec * const ehp) { + RUN_HOST_ONLY(); cq_status status = CQ_ERROR; char * fname = NULL; @@ -127,8 +142,12 @@ cstate * crp, const size_t NMEASURE, cq_exec * const ehp) { return status; } -cq_status sm_qrun(qkern kernel, qubit * qrp, const size_t NQUBITS, -cstate * const crp, const size_t NMEASURE, const size_t NSHOTS) { +cq_status sm_qrun(qkern kernel, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const size_t NSHOTS) { cq_exec exec_handle; cq_status status = CQ_ERROR; @@ -145,9 +164,14 @@ cstate * const crp, const size_t NMEASURE, const size_t NSHOTS) { return status; } -cq_status am_qrun(qkern kernel, qubit * qrp, const size_t NQUBITS, -cstate * const crp, const size_t NMEASURE, const size_t NSHOTS, -cq_exec * const ehp) { +cq_status am_qrun(qkern kernel, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const size_t NSHOTS, + cq_exec * const ehp) { + RUN_HOST_ONLY(); cq_status status = CQ_ERROR; char * fname = NULL; @@ -177,21 +201,155 @@ cq_exec * const ehp) { return status; } +cq_status sp_qrun(pqkern kernel, + void * kernpar, + const size_t KERNPAR_SIZE, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE) { + RUN_HOST_ONLY(); + cq_status status = CQ_ERROR; + cq_exec exec_handle; + + status = ap_qrun(kernel, kernpar, KERNPAR_SIZE, qrp, NQUBITS, crp, NMEASURE, + &exec_handle); + + if (status == CQ_SUCCESS) { + status = wait_qrun(&exec_handle); + + if (status == CQ_SUCCESS) { + status = exec_handle.status; + } + } + + return status; +} + +cq_status ap_qrun(pqkern kernel, + void * kernpar, + const size_t KERNPAR_SIZE, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + cq_exec * const ehp) { + RUN_HOST_ONLY(); + cq_status status = CQ_ERROR; + char * fname = NULL; + + if (ehp == NULL) return status; + init_exec_handle(NQUBITS, 1, NMEASURE, ehp); + + if (KERNPAR_SIZE < 0) return status; + ehp->params = kernpar; + ehp->params_size = KERNPAR_SIZE; + + if (qrp != NULL && (NMEASURE == 0 || crp != NULL)) { + status = find_pqkern_name(kernel, &fname); + + if (status == CQ_SUCCESS) { + ehp->fname = fname; + ehp->qreg = qrp; + ehp->creg = crp; + + host_send_ctrl_op(CQ_CTRL_RUN_PQKERNEL, ehp); + } else { + finalise_exec_handle(ehp); + } + } else { + finalise_exec_handle(ehp); + } + + return status; +} + +cq_status smp_qrun(pqkern kernel, + void * kernpar, + const size_t KERNPAR_SIZE, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const size_t NSHOTS) { + cq_exec exec_handle; + cq_status status = CQ_ERROR; + + status = amp_qrun(kernel, kernpar, KERNPAR_SIZE, qrp, NQUBITS, crp, NMEASURE, + NSHOTS, &exec_handle); + + if (status == CQ_SUCCESS) { + status = wait_qrun(&exec_handle); + + if (status == CQ_SUCCESS) { + status = exec_handle.status; + } + } + + return status; +} + +cq_status amp_qrun(pqkern kernel, + void * kernpar, + const size_t KERNPAR_SIZE, + qubit * qrp, + const size_t NQUBITS, + cstate * const crp, + const size_t NMEASURE, + const size_t NSHOTS, + cq_exec * const ehp) { + RUN_HOST_ONLY(); + cq_status status = CQ_ERROR; + char * fname = NULL; + + if (ehp == NULL) return status; + init_exec_handle(NQUBITS, NSHOTS, NMEASURE, ehp); + + if (KERNPAR_SIZE < 0) return status; + ehp->params = kernpar; + ehp->params_size = KERNPAR_SIZE; + + if (NSHOTS == 0) { + status = CQ_SUCCESS; + finalise_exec_handle(ehp); + } else if (qrp != NULL && (NMEASURE == 0 || crp != NULL)) { + status = find_pqkern_name(kernel, &fname); + + if (status == CQ_SUCCESS) { + ehp->fname = fname; + ehp->qreg = qrp; + ehp->creg = crp; + host_send_ctrl_op(CQ_CTRL_RUN_PQKERNEL, ehp); + } else { + finalise_exec_handle(ehp); + } + + } else { + finalise_exec_handle(ehp); + } + + return status; +} + // Synchronisation cq_status sync_qrun(cq_exec * const ehp) { + RUN_HOST_ONLY(); cq_status status = CQ_ERROR; if (ehp != NULL && ehp->exec_init) { - host_sync_exec(ehp); + // host_sync_exec(ehp); + host_send_ctrl_op(CQ_CTRL_SYNC_EXEC, ehp); status = CQ_SUCCESS; } - return status; + return status; } cq_status wait_qrun(cq_exec * const ehp) { + RUN_HOST_ONLY(); cq_status status = CQ_ERROR; if (ehp != NULL && ehp->exec_init) { - host_wait_exec(ehp); + // host_wait_exec(ehp); + host_send_ctrl_op(CQ_CTRL_WAIT_EXEC, ehp); finalise_exec_handle(ehp); status = CQ_SUCCESS; } else if (ehp != NULL && ehp->expected_shots == 0) { @@ -202,9 +360,11 @@ cq_status wait_qrun(cq_exec * const ehp) { } cq_status halt_qrun(cq_exec * const ehp) { + RUN_HOST_ONLY(); cq_status status = CQ_ERROR; if (ehp != NULL && ehp->exec_init) { - host_request_halt(ehp); + // host_request_halt(ehp); + host_send_ctrl_op(CQ_CTRL_ABORT, ehp); status = wait_qrun(ehp); } return status; diff --git a/src/host/opcodes.h b/src/host/opcodes.h index b723065..c8965e5 100644 --- a/src/host/opcodes.h +++ b/src/host/opcodes.h @@ -10,7 +10,10 @@ enum ctrl_code { CQ_CTRL_DEALLOC, CQ_CTRL_RUN_QKERNEL, CQ_CTRL_RUN_PQKERNEL, - CQ_CTRL_TEST + CQ_CTRL_TEST, + CQ_CTRL_WAIT, // added in MPI extension + CQ_CTRL_WAIT_EXEC, // ditto + CQ_CTRL_SYNC_EXEC, // ditto }; -#endif \ No newline at end of file +#endif diff --git a/tests/fortran/CMakeLists.txt b/tests/fortran/CMakeLists.txt index 709e9a7..8089e25 100644 --- a/tests/fortran/CMakeLists.txt +++ b/tests/fortran/CMakeLists.txt @@ -8,6 +8,10 @@ add_executable(test_fort_host_ops utils.f90 ) +set_target_properties(test_fort_host_ops PROPERTIES + Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/mod/test_fort_host_ops" +) + target_link_libraries(test_fort_host_ops PRIVATE cq-simbe @@ -23,6 +27,10 @@ add_executable(test_fort_device_ops utils.f90 ) +set_target_properties(test_fort_device_ops PROPERTIES + Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/mod/test_fort_device_ops" +) + target_link_libraries(test_fort_device_ops PRIVATE cq-simbe @@ -38,6 +46,10 @@ add_executable(test_fort_qasm_gates utils.f90 ) +set_target_properties(test_fort_qasm_gates PROPERTIES + Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/mod/test_fort_qasm_gates" +) + target_link_libraries(test_fort_qasm_gates PRIVATE cq-simbe