diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 11c2e527f3..b18ad50c25 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -100,14 +100,44 @@ jobs: - name: Prepare for testing run: | brew services restart postgresql@14 + for _ in {1..30}; do + if pg_isready -h 127.0.0.1 -p 5432 -U postgres; then + break + fi + sleep 1 + done + pg_isready -h 127.0.0.1 -p 5432 -U postgres + brew services start mariadb + for _ in {1..30}; do + if mariadb-admin ping --silent; then + break + fi + sleep 1 + done + mariadb-admin ping --silent + brew services start redis - sleep 4 + for _ in {1..30}; do + if redis-cli ping | grep -q PONG; then + break + fi + sleep 1 + done + redis-cli ping | grep -q PONG + mariadb -e "SET PASSWORD FOR 'root'@'localhost' = PASSWORD('')" mariadb -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost'" mariadb -e "FLUSH PRIVILEGES" brew services restart mariadb - sleep 4 + for _ in {1..30}; do + if mariadb-admin ping --silent; then + break + fi + sleep 1 + done + mariadb-admin ping --silent + psql -c 'create user postgres superuser;' postgres - name: Test @@ -228,7 +258,13 @@ jobs: - name: Prepare for testing run: | sudo systemctl start postgresql - sleep 1 + for _ in {1..30}; do + if pg_isready -h 127.0.0.1 -p 5432 -U postgres; then + break + fi + sleep 1 + done + pg_isready -h 127.0.0.1 -p 5432 -U postgres sudo -u postgres psql -c "ALTER USER postgres WITH PASSWORD '12345'" postgres - name: Test diff --git a/lib/tests/integration_test/client/RequestStreamTest.cc b/lib/tests/integration_test/client/RequestStreamTest.cc index 8fea7cf2c2..94bdee20ad 100644 --- a/lib/tests/integration_test/client/RequestStreamTest.cc +++ b/lib/tests/integration_test/client/RequestStreamTest.cc @@ -5,6 +5,7 @@ #include #include #include +#include using namespace drogon; @@ -100,23 +101,46 @@ DROGON_TEST(RequestStreamTest) LOG_INFO << "Test request stream"; - std::string filePath = "./中文.txt"; - std::ifstream file(filePath); - std::stringstream content; - REQUIRE(file.is_open()); - content << file.rdbuf(); + const auto uniqueSuffix = std::to_string( + std::chrono::steady_clock::now().time_since_epoch().count()); + auto tempDir = std::make_shared( + std::filesystem::temp_directory_path() / + ("request_stream_upload_test_" + uniqueSuffix)); + std::filesystem::create_directories(*tempDir); + auto tempPath = std::make_shared( + *tempDir / std::filesystem::path(u8"中文.txt")); + tempPath->make_preferred(); + { + std::ofstream out(*tempPath, std::ios::binary | std::ios::trunc); + REQUIRE(out.is_open()); + out << "request-stream-upload-content\nline2\n"; + } + + std::ifstream in(*tempPath, std::ios::binary); + REQUIRE(in.is_open()); + std::stringstream ss; + ss << in.rdbuf(); + const auto uploadContent = std::make_shared(ss.str()); - req = HttpRequest::newFileUploadRequest({UploadFile{filePath}}); + const auto uploadPathUtf8 = std::make_shared([&tempPath]() { + auto u8Path = tempPath->u8string(); + return std::string(reinterpret_cast(u8Path.data()), + u8Path.size()); + }()); + req = HttpRequest::newFileUploadRequest({UploadFile{*uploadPathUtf8}}); req->setPath("/stream_upload_echo"); req->setMethod(Post); - client->sendRequest(req, - [TEST_CTX, - content = content.str()](ReqResult r, - const HttpResponsePtr &resp) { - CHECK(r == ReqResult::Ok); - CHECK(resp->statusCode() == k200OK); - CHECK(resp->body() == content); - }); + client->sendRequest( + req, + [TEST_CTX, tempPath, tempDir, uploadPathUtf8, content = uploadContent]( + ReqResult r, const HttpResponsePtr &resp) { + CHECK(r == ReqResult::Ok); + CHECK(resp->statusCode() == k200OK); + CHECK(resp->body() == *content); + std::error_code ec; + std::filesystem::remove(*tempPath, ec); + std::filesystem::remove(*tempDir, ec); + }); checkStreamRequest(TEST_CTX, client->getLoop(), diff --git a/orm_lib/src/TransactionImpl.cc b/orm_lib/src/TransactionImpl.cc index d63fc688c8..471b6a893a 100644 --- a/orm_lib/src/TransactionImpl.cc +++ b/orm_lib/src/TransactionImpl.cc @@ -205,6 +205,8 @@ void TransactionImpl::execNewTask() { loop_->assertInLoopThread(); thisPtr_.reset(); + if (!isWorking_) + return; assert(isWorking_); if (!isCommitedOrRolledback_) { @@ -246,23 +248,31 @@ void TransactionImpl::execNewTask() else { isWorking_ = false; - if (!sqlCmdBuffer_.empty()) - { - auto exceptPtr = std::make_exception_ptr( - TransactionRollback("The transaction has been rolled back")); - for (auto const &cmd : sqlCmdBuffer_) - { - if (cmd->exceptionCallback_) - { - cmd->exceptionCallback_(exceptPtr); - } - } - sqlCmdBuffer_.clear(); - } - if (usedUpCallback_) + failBufferedCommands(std::make_exception_ptr( + TransactionRollback("The transaction has been rolled back"))); + releaseConnection(); + } +} + +void TransactionImpl::releaseConnection() +{ + if (usedUpCallback_) + { + usedUpCallback_(); + usedUpCallback_ = std::function(); + } +} + +void TransactionImpl::failBufferedCommands(const std::exception_ptr &ePtr) +{ + std::list pendingCmds; + pendingCmds.swap(sqlCmdBuffer_); + for (auto &cmd : pendingCmds) + { + cmd->thisPtr_.reset(); + if (cmd->exceptionCallback_) { - usedUpCallback_(); - usedUpCallback_ = std::function(); + cmd->exceptionCallback_(ePtr); } } } @@ -307,6 +317,12 @@ void TransactionImpl::doBegin() [thisPtr](const std::exception_ptr &) { LOG_ERROR << "Error occurred in transaction begin"; thisPtr->isCommitedOrRolledback_ = true; + thisPtr->isWorking_ = false; + thisPtr->thisPtr_.reset(); + thisPtr->failBufferedCommands(std::make_exception_ptr( + TransactionRollback("Transaction begin failed, cannot " + "execute queued SQL"))); + thisPtr->releaseConnection(); }); }); } diff --git a/orm_lib/src/TransactionImpl.h b/orm_lib/src/TransactionImpl.h index 528f023e9b..441ef6b061 100644 --- a/orm_lib/src/TransactionImpl.h +++ b/orm_lib/src/TransactionImpl.h @@ -132,6 +132,8 @@ class TransactionImpl : public Transaction, bool isCommitedOrRolledback_{false}; bool isWorking_{false}; void execNewTask(); + void releaseConnection(); + void failBufferedCommands(const std::exception_ptr &ePtr); struct SqlCmd { diff --git a/test.sh b/test.sh index 4796ff11a1..f99b9cadb3 100755 --- a/test.sh +++ b/test.sh @@ -23,46 +23,147 @@ else fi echo "drogon_ctl_exec: " ${drogon_ctl_exec} -#Make integration_test_server run as a daemon -function do_integration_test() +if [ "X$os" = "Xwindows" ]; then + integration_test_client_exec=./integration_test_client.exe + integration_test_server_exec=./integration_test_server.exe +else + integration_test_client_exec=./integration_test_client + integration_test_server_exec=./integration_test_server +fi + +function update_config_line() { - pushd $test_root - if [ "X$os" = "Xlinux" ]; then - sed -i -e "s/\"run_as_daemon.*$/\"run_as_daemon\": true\,/" config.example.json + local key="$1" + local value="$2" + local file="$3" + sed -i.bak -e "s/\"${key}\".*$/\"${key}\": ${value},/" "$file" + rm -f "$file.bak" +} + +function cleanup_integration_test_server() +{ + if [ "X$os" = "Xwindows" ]; then + taskkill //F //IM integration_test_server.exe > /dev/null 2>&1 || true + else + killall integration_test_server > /dev/null 2>&1 || true + pkill -f '/integration_test_server$' > /dev/null 2>&1 || true + fi +} + +function wait_for_url() +{ + local url="$1" + local timeout_seconds="$2" + local curl_args=(--silent --show-error --output /dev/null --max-time 2) + + if [ "$url" != "${url#https://}" ]; then + curl_args+=(--insecure) + fi + + local attempt=0 + while [ $attempt -lt $timeout_seconds ]; do + if curl "${curl_args[@]}" "$url"; then + return 0 + fi + + attempt=$((attempt + 1)) + sleep 1 + done + + return 1 +} + +function wait_for_integration_test_server() +{ + local server_pid="$1" + local server_log="$2" + + if ! wait_for_url "http://127.0.0.1:8848/" 30; then + echo "Timed out waiting for integration_test_server to accept HTTP requests" + if kill -0 "$server_pid" > /dev/null 2>&1; then + echo "integration_test_server is still running, recent log output:" + else + echo "integration_test_server exited before becoming ready, recent log output:" + fi + if [ -f "$server_log" ]; then + tail -n 50 "$server_log" + fi + return 1 + fi + + wait_for_url "https://127.0.0.1:8849/" 5 > /dev/null 2>&1 || true + return 0 +} + +function get_cpu_count() +{ + if command -v nproc > /dev/null 2>&1; then + nproc + return fi - sed -i -e "s/\"relaunch_on_error.*$/\"relaunch_on_error\": true\,/" config.example.json - sed -i -e "s/\"threads_num.*$/\"threads_num\": 0\,/" config.example.json - sed -i -e "s/\"use_brotli.*$/\"use_brotli\": true\,/" config.example.json + + if command -v getconf > /dev/null 2>&1; then + getconf _NPROCESSORS_ONLN + return + fi + + if command -v sysctl > /dev/null 2>&1; then + sysctl -n hw.logicalcpu + return + fi + + echo 1 +} + +trap cleanup_integration_test_server EXIT + +# Run the integration test server in the background and wait until it is ready. +function do_integration_test() +{ + pushd "$test_root" + update_config_line "run_as_daemon" "false" config.example.json + update_config_line "relaunch_on_error" "false" config.example.json + update_config_line "number_of_threads" "1" config.example.json + update_config_line "use_brotli" "true" config.example.json if [ "$1" = "stream_mode" ]; then - sed -i -e "s/\"enable_request_stream.*$/\"enable_request_stream\": true\,/" config.example.json + update_config_line "enable_request_stream" "true" config.example.json else - sed -i -e "s/\"enable_request_stream.*$/\"enable_request_stream\": false\,/" config.example.json + update_config_line "enable_request_stream" "false" config.example.json fi - if [ ! -f "integration_test_client" ]; then + if [ ! -f "$integration_test_client_exec" ]; then echo "Build failed" exit -1 fi - if [ ! -f "integration_test_server" ]; then + if [ ! -f "$integration_test_server_exec" ]; then echo "Build failed" exit -1 fi - killall -9 integration_test_server - ./integration_test_server & + cleanup_integration_test_server + + local server_log=integration_test_server.log + rm -f "$server_log" + "$integration_test_server_exec" > "$server_log" 2>&1 & + local server_pid=$! - sleep 4 + if ! wait_for_integration_test_server "$server_pid" "$server_log"; then + exit -1 + fi echo "Running the integration test $1" - ./integration_test_client -s + "$integration_test_client_exec" -s if [ $? -ne 0 ]; then echo "Integration test failed $1" + if [ -f "$server_log" ]; then + tail -n 50 "$server_log" + fi exit -1 fi - killall -9 integration_test_server + cleanup_integration_test_server popd } @@ -70,7 +171,7 @@ function do_integration_test() function do_drogon_ctl_test() { echo "Testing drogon_ctl" - pushd $test_root + pushd "$test_root" rm -rf drogon_test ${drogon_ctl_exec} create project drogon_test @@ -122,22 +223,23 @@ function do_drogon_ctl_test() make_flags='' cmake_gen='' parallel=1 + cpu_count=$(get_cpu_count) # simulate ninja's parallelism - case $(nproc) in + case $cpu_count in 1) - parallel=$(($(nproc) + 1)) + parallel=$((cpu_count + 1)) ;; 2) - parallel=$(($(nproc) + 1)) + parallel=$((cpu_count + 1)) ;; *) - parallel=$(($(nproc) + 2)) + parallel=$((cpu_count + 2)) ;; esac if [ "X$os" = "Xlinux" ]; then - if [ -f /bin/ninja ]; then + if command -v ninja > /dev/null 2>&1; then cmake_gen='-G Ninja' else make_flags="$make_flags -j$parallel" @@ -162,11 +264,11 @@ function do_drogon_ctl_test() exit -1 fi - if [ "X$os" = "Xlinux" ]; then - if [ ! -f "drogon_test" ]; then - echo "Failed to build drogon_test" - exit -1 - fi + if [ "X$os" = "Xlinux" ]; then + if [ ! -f "drogon_test" ]; then + echo "Failed to build drogon_test" + exit -1 + fi else if [ ! -f "Debug\drogon_test.exe" ]; then echo "Failed to build drogon_test" @@ -183,7 +285,7 @@ function do_drogon_ctl_test() function do_unittest() { echo "Unit testing" - pushd $src_dir/build + pushd "$src_dir/build" ctest . --output-on-failure if [ $? -ne 0 ]; then @@ -195,7 +297,7 @@ function do_unittest() function do_db_test() { - pushd $src_dir/build + pushd "$src_dir/build" if [ -f "./orm_lib/tests/db_test" ]; then echo "Test database" ./orm_lib/tests/db_test -s @@ -246,9 +348,9 @@ function do_db_test() fi } -if ! drogon_ctl -v > /dev/null 2>&1 +if [ ! -f "$drogon_ctl_exec" ] then - echo "Warning: No drogon_ctl, skip integration test and drogon_ctl test" + echo "Warning: No built drogon_ctl, skip integration test and drogon_ctl test" else do_integration_test do_integration_test stream_mode