diff --git a/.gitignore b/.gitignore index 76d5505..9fd4727 100644 --- a/.gitignore +++ b/.gitignore @@ -106,3 +106,7 @@ Desktop.ini *.log *.sql *.sqlite +*.sln +*.vcxproj + +*.mkv diff --git a/README.md b/README.md index fb79667..3672854 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The instructions below are based on the [Azure Kinect Sensor SDK Usage](https:// * Install the [Azure Kinect Sensor SDK](https://docs.microsoft.com/en-us/azure/Kinect-dk/sensor-sdk-download). * Install the [Azure Kinect Body Tracking SDK](https://docs.microsoft.com/en-us/azure/Kinect-dk/body-sdk-download). -* Add an environment variable for `AZUREKINECT_SDK` and set it to the Sensor SDK installation path (no trailing slash). The default is `C:\Program Files\Azure Kinect SDK v1.3.0`. +* Add an environment variable for `AZUREKINECT_SDK` and set it to the Sensor SDK installation path (no trailing slash). The default is `C:\Program Files\Azure Kinect SDK v1.4.1`. * Add an environment variable for `AZUREKINECT_BODY_SDK` and set it to the Body SDK installation path (no trailing slash). The default is `C:\Program Files\Azure Kinect Body Tracking SDK`. ![Environment Variables](Install-EnvVars.png) diff --git a/addon_config.mk b/addon_config.mk index c9e14f5..d080a15 100644 --- a/addon_config.mk +++ b/addon_config.mk @@ -64,6 +64,7 @@ vs: ADDON_INCLUDES += $(AZUREKINECT_SDK)\sdk\include ADDON_INCLUDES += $(AZUREKINECT_BODY_SDK)\sdk\include ADDON_LIBS += $(AZUREKINECT_SDK)\sdk\windows-desktop\amd64\release\lib\k4a.lib + ADDON_LIBS += $(AZUREKINECT_SDK)\sdk\windows-desktop\amd64\release\lib\k4arecord.lib ADDON_LIBS += $(AZUREKINECT_BODY_SDK)\sdk\windows-desktop\amd64\release\lib\k4abt.lib linux64: @@ -71,8 +72,10 @@ linux64: ADDON_INCLUDES += /usr/include/k4a ADDON_LIBS += /usr/lib/libk4abt.so ADDON_LIBS += /usr/lib/x86_64-linux-gnu/libk4a.so - ADDON_LIBS += /opt/libjpeg-turbo/lib64/libturbojpeg.a - + # ADDON_LIBS += /opt/libjpeg-turbo/lib64/libturbojpeg.a + ADDON_LIBS += /usr/lib/x86_64-linux-gnu/libturbojpeg.so.0 + ADDON_LDFLAGS += -lk4arecord + linux: linuxarmv6l: diff --git a/example-bodies/example-bodies.vcxproj b/example-bodies/example-bodies.vcxproj index 8d8ad04..e3c141e 100644 --- a/example-bodies/example-bodies.vcxproj +++ b/example-bodies/example-bodies.vcxproj @@ -100,7 +100,7 @@ %(PreprocessorDefinitions) MultiThreadedDebugDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp $(IntDir) @@ -108,8 +108,8 @@ true Console false - %(AdditionalDependencies);k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -120,7 +120,7 @@ %(PreprocessorDefinitions) MultiThreadedDebugDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp true $(IntDir) @@ -129,8 +129,8 @@ true Console false - %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -140,7 +140,7 @@ %(PreprocessorDefinitions) MultiThreadedDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp true $(IntDir) @@ -152,8 +152,8 @@ true true false - %(AdditionalDependencies);k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -163,7 +163,7 @@ %(PreprocessorDefinitions) MultiThreadedDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp $(IntDir) @@ -174,19 +174,25 @@ true true false - %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + + + + + + diff --git a/example-bodies/example-bodies.vcxproj.filters b/example-bodies/example-bodies.vcxproj.filters index df9faff..d599c86 100644 --- a/example-bodies/example-bodies.vcxproj.filters +++ b/example-bodies/example-bodies.vcxproj.filters @@ -13,9 +13,18 @@ src + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + @@ -50,9 +59,18 @@ src + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect diff --git a/example-bodies/src/ofApp.cpp b/example-bodies/src/ofApp.cpp index c7eef9f..af62460 100644 --- a/example-bodies/src/ofApp.cpp +++ b/example-bodies/src/ofApp.cpp @@ -7,36 +7,39 @@ void ofApp::setup() ofLogNotice(__FUNCTION__) << "Found " << ofxAzureKinect::Device::getInstalledCount() << " installed devices."; - auto deviceSettings = ofxAzureKinect::DeviceSettings(); - deviceSettings.syncImages = false; - deviceSettings.depthMode = K4A_DEPTH_MODE_NFOV_UNBINNED; - deviceSettings.updateIr = false; - deviceSettings.updateColor = false; - //deviceSettings.colorResolution = K4A_COLOR_RESOLUTION_1080P; - deviceSettings.updateWorld = true; - deviceSettings.updateVbo = false; + // Load Body Tracking Settings auto bodyTrackingSettings = ofxAzureKinect::BodyTrackingSettings(); - //bodyTrackingSettings.processingMode = K4ABT_TRACKER_PROCESSING_MODE_CPU; bodyTrackingSettings.updateBodies = true; - if (this->kinectDevice.open()) + + // Load a recording with Body Tracking Settings + if (!streaming) { - this->kinectDevice.startCameras(deviceSettings, bodyTrackingSettings); + filename = ofToDataPath("output_2d_movements.mkv"); + if (!kinectDevice.load(filename)) + { + exit(); + } + // Start Playback or Streaming + kinectDevice.startCameras(); } - - // Load shader. - auto shaderSettings = ofShaderSettings(); - shaderSettings.shaderFiles[GL_VERTEX_SHADER] = "shaders/render.vert"; - shaderSettings.shaderFiles[GL_FRAGMENT_SHADER] = "shaders/render.frag"; - shaderSettings.intDefines["BODY_INDEX_MAP_BACKGROUND"] = K4ABT_BODY_INDEX_MAP_BACKGROUND; - shaderSettings.bindDefaults = true; - if (this->shader.setup(shaderSettings)) + else { - ofLogNotice(__FUNCTION__) << "Success loading shader!"; + auto deviceSettings = ofxAzureKinect::DeviceSettings(); + deviceSettings.syncImages = false; + deviceSettings.depthMode = K4A_DEPTH_MODE_NFOV_UNBINNED; + deviceSettings.updateIr = false; + deviceSettings.updateColor = false; + //deviceSettings.colorResolution = K4A_COLOR_RESOLUTION_1080P; + deviceSettings.updateWorld = true; + deviceSettings.updateVbo = false; + + if (!kinectDevice.open()) + { + exit(); + } + // Start Playback or Streaming + kinectDevice.startCameras(deviceSettings, bodyTrackingSettings); } - - // Setup vbo. - std::vector verts(1); - this->pointsVbo.setVertexData(verts.data(), verts.size(), GL_STATIC_DRAW); } //-------------------------------------------------------------- @@ -48,7 +51,6 @@ void ofApp::exit() //-------------------------------------------------------------- void ofApp::update() { - } //-------------------------------------------------------------- @@ -56,178 +58,77 @@ void ofApp::draw() { ofBackground(0); - if (this->kinectDevice.isStreaming()) - { - this->kinectDevice.getBodyIndexTex().draw(0, 0, 360, 360); - } + ofxAzureKinect::BodyTracker *tracker = kinectDevice.getBodyTracker(); - this->camera.begin(); - { - ofPushMatrix(); - { - ofRotateXDeg(180); - - ofEnableDepthTest(); + tracker->draw_body_map(); - constexpr int kMaxBodies = 6; - int bodyIDs[kMaxBodies]; - int i = 0; - while (i < this->kinectDevice.getNumBodies()) - { - bodyIDs[i] = this->kinectDevice.getBodyIDs()[i]; - ++i; - } - while (i < kMaxBodies) - { - bodyIDs[i] = 0; - ++i; - } + camera.begin(); - this->shader.begin(); - { - this->shader.setUniformTexture("uDepthTex", this->kinectDevice.getDepthTex(), 1); - this->shader.setUniformTexture("uBodyIndexTex", this->kinectDevice.getBodyIndexTex(), 2); - this->shader.setUniformTexture("uWorldTex", this->kinectDevice.getDepthToWorldTex(), 3); - this->shader.setUniform2i("uFrameSize", this->kinectDevice.getDepthTex().getWidth(), this->kinectDevice.getDepthTex().getHeight()); - this->shader.setUniform1iv("uBodyIDs", bodyIDs, kMaxBodies); + ofPushMatrix(); + ofRotateXDeg(180); + tracker->draw_point_clouds(kinectDevice.getDepthTex(), kinectDevice.getDepthToWorldTex()); + tracker->draw_skeletons(); + ofPopMatrix(); - int numPoints = this->kinectDevice.getDepthTex().getWidth() * this->kinectDevice.getDepthTex().getHeight(); - this->pointsVbo.drawInstanced(GL_POINTS, 0, 1, numPoints); - } - this->shader.end(); + camera.end(); - ofDisableDepthTest(); + stringstream ss; + ss << ofToString(ofGetFrameRate(), 2) + " FPS" << std::endl; + ss << "Joint Smoothing: " << tracker->get_joint_smoothing(); + ofDrawBitmapStringHighlight(ss.str(), 10, 20); +} - auto& bodySkeletons = this->kinectDevice.getBodySkeletons(); - for (auto& skeleton : bodySkeletons) +//-------------------------------------------------------------- +void ofApp::keyPressed(int key) +{ + // handle playback hotkeys + if (!streaming) + { + switch (key) + { + case ' ': + { + play = !play; + if (play) { - // Draw joints. - for (int i = 0; i < K4ABT_JOINT_COUNT; ++i) - { - auto joint = skeleton.joints[i]; - ofPushMatrix(); - { - glm::mat4 transform = glm::translate(toGlm(joint.position)) * glm::toMat4(toGlm(joint.orientation)); - ofMultMatrix(transform); - - ofDrawAxis(50.0f); - } - ofPopMatrix(); - } - - // Draw connections. - this->skeletonMesh.setMode(OF_PRIMITIVE_LINES); - auto& vertices = this->skeletonMesh.getVertices(); - vertices.resize(50); - int vdx = 0; - - // Spine. - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_PELVIS].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SPINE_NAVEL].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SPINE_NAVEL].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SPINE_CHEST].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SPINE_CHEST].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NECK].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NECK].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_HEAD].position); - - // Head. - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_HEAD].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NOSE].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NOSE].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_EYE_LEFT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_EYE_LEFT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_EAR_LEFT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NOSE].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_EYE_RIGHT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_EYE_RIGHT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_EAR_RIGHT].position); - - // Left Leg. - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_PELVIS].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_HIP_LEFT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_HIP_LEFT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_KNEE_LEFT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_KNEE_LEFT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ANKLE_LEFT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ANKLE_LEFT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_FOOT_LEFT].position); - - // Right leg. - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_PELVIS].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_HIP_RIGHT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_HIP_RIGHT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_KNEE_RIGHT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_KNEE_RIGHT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ANKLE_RIGHT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ANKLE_RIGHT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_FOOT_RIGHT].position); - - // Left arm. - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NECK].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_CLAVICLE_LEFT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_CLAVICLE_LEFT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SHOULDER_LEFT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SHOULDER_LEFT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ELBOW_LEFT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ELBOW_LEFT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_WRIST_LEFT].position); - - // Right arm. - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NECK].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_CLAVICLE_RIGHT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_CLAVICLE_RIGHT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SHOULDER_RIGHT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SHOULDER_RIGHT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ELBOW_RIGHT].position); - - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ELBOW_RIGHT].position); - vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_WRIST_RIGHT].position); - - this->skeletonMesh.draw(); + kinectDevice.play = true; } + else + { + kinectDevice.pause = false; + } + break; + } + case '+': + play_head = ofClamp(play_head + .01, 0, 1); + kinectDevice.seek = play_head; + break; + case '-': + play_head = ofClamp(play_head - .01, 0, 1); + kinectDevice.seek = play_head; + break; } - ofPopMatrix(); - } - this->camera.end(); - - std::ostringstream oss; - oss << ofToString(ofGetFrameRate(), 2) + " FPS" << std::endl; - oss << "Joint Smoothing: " << this->kinectDevice.jointSmoothing; - ofDrawBitmapStringHighlight(oss.str(), 10, 20); -} - -//-------------------------------------------------------------- -void ofApp::keyPressed(int key){ + switch (key) + { + case 'f': + case 'F': + ofToggleFullscreen(); + break; + default: + break; + } + } } //-------------------------------------------------------------- -void ofApp::keyReleased(int key){ - +void ofApp::keyReleased(int key) +{ } //-------------------------------------------------------------- -void ofApp::mouseMoved(int x, int y ){ - +void ofApp::mouseMoved(int x, int y) +{ } //-------------------------------------------------------------- @@ -235,41 +136,41 @@ void ofApp::mouseDragged(int x, int y, int button) { if (button == 1) { - this->kinectDevice.jointSmoothing = ofMap(x, 0, ofGetWidth(), 0.0f, 1.0f, true); + this->kinectDevice.getBodyTracker()->set_joint_smoothing(ofMap(x, 0, ofGetWidth(), 0.0f, 1.0f, true)); } } //-------------------------------------------------------------- -void ofApp::mousePressed(int x, int y, int button){ - +void ofApp::mousePressed(int x, int y, int button) +{ } //-------------------------------------------------------------- -void ofApp::mouseReleased(int x, int y, int button){ - +void ofApp::mouseReleased(int x, int y, int button) +{ } //-------------------------------------------------------------- -void ofApp::mouseEntered(int x, int y){ - +void ofApp::mouseEntered(int x, int y) +{ } //-------------------------------------------------------------- -void ofApp::mouseExited(int x, int y){ - +void ofApp::mouseExited(int x, int y) +{ } //-------------------------------------------------------------- -void ofApp::windowResized(int w, int h){ - +void ofApp::windowResized(int w, int h) +{ } //-------------------------------------------------------------- -void ofApp::gotMessage(ofMessage msg){ - +void ofApp::gotMessage(ofMessage msg) +{ } //-------------------------------------------------------------- -void ofApp::dragEvent(ofDragInfo dragInfo){ - +void ofApp::dragEvent(ofDragInfo dragInfo) +{ } diff --git a/example-bodies/src/ofApp.h b/example-bodies/src/ofApp.h index 4d5e0c2..9cc8abe 100644 --- a/example-bodies/src/ofApp.h +++ b/example-bodies/src/ofApp.h @@ -4,8 +4,8 @@ #include "ofxAzureKinect.h" -class ofApp - : public ofBaseApp +class ofApp + : public ofBaseApp { public: void setup(); @@ -31,8 +31,11 @@ class ofApp ofEasyCam camera; - ofVbo pointsVbo; - ofShader shader; + bool streaming = true; + + // Playback Params + string filename; + bool play = false; + float play_head = 0; - ofVboMesh skeletonMesh; }; diff --git a/example-multi/example-multi.vcxproj b/example-multi/example-multi.vcxproj index 9bd23de..3e3b7e8 100644 --- a/example-multi/example-multi.vcxproj +++ b/example-multi/example-multi.vcxproj @@ -100,7 +100,7 @@ %(PreprocessorDefinitions) MultiThreadedDebugDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp $(IntDir) @@ -108,8 +108,8 @@ true Console false - %(AdditionalDependencies);k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -120,7 +120,7 @@ %(PreprocessorDefinitions) MultiThreadedDebugDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp true $(IntDir) @@ -129,8 +129,8 @@ true Console false - %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -140,7 +140,7 @@ %(PreprocessorDefinitions) MultiThreadedDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp true $(IntDir) @@ -152,8 +152,8 @@ true true false - %(AdditionalDependencies);k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -163,7 +163,7 @@ %(PreprocessorDefinitions) MultiThreadedDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp $(IntDir) @@ -174,19 +174,25 @@ true true false - %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + + + + + + diff --git a/example-multi/example-multi.vcxproj.filters b/example-multi/example-multi.vcxproj.filters index df9faff..d599c86 100644 --- a/example-multi/example-multi.vcxproj.filters +++ b/example-multi/example-multi.vcxproj.filters @@ -13,9 +13,18 @@ src + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + @@ -50,9 +59,18 @@ src + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect diff --git a/example-multi/src/main.cpp b/example-multi/src/main.cpp index aaae62e..249044c 100644 --- a/example-multi/src/main.cpp +++ b/example-multi/src/main.cpp @@ -4,7 +4,7 @@ int main() { ofGLFWWindowSettings settings; settings.setGLVersion(3, 2); - settings.setSize(640 * 2, 360 + 320); + settings.setSize(640 * 3, 360 + 320); ofCreateWindow(settings); ofRunApp(new ofApp()); diff --git a/example-multi/src/ofApp.cpp b/example-multi/src/ofApp.cpp index 7a74815..d4859ed 100644 --- a/example-multi/src/ofApp.cpp +++ b/example-multi/src/ofApp.cpp @@ -43,44 +43,70 @@ void ofApp::setupStandalone() //-------------------------------------------------------------- void ofApp::setupMasterSubordinate() { - // Make sure to replace the following serials by the ones on your devices. - const std::string serialMaster = "000224694712"; - const std::string serialSubordinate = "000569192412"; - auto kinectSettings = ofxAzureKinect::DeviceSettings(); - kinectSettings.colorResolution = K4A_COLOR_RESOLUTION_720P; + kinectSettings.colorResolution = K4A_COLOR_RESOLUTION_1080P; kinectSettings.syncImages = true; kinectSettings.updateWorld = false; - // Open Master device. + // Open all device and check sync status. + int numConnected = ofxAzureKinect::Device::getInstalledCount(); + int syncOutDeviceIndex = -1; + int syncInOutDeviceIndex = -1; + + int connectedDeviceIndex = 0; + for (int i = 0; i < numConnected; ++i) { auto device = std::make_shared(); - if (device->open(serialMaster)) + if (device->open(i)) { - kinectSettings.wiredSyncMode = K4A_WIRED_SYNC_MODE_MASTER; - device->startCameras(kinectSettings); - this->kinectDevices.push_back(device); + if (!device->isSyncInConnected() && device->isSyncOutConnected()) { + syncOutDeviceIndex = connectedDeviceIndex; + } + else if (device->isSyncInConnected() && device->isSyncOutConnected()) { + syncInOutDeviceIndex = connectedDeviceIndex; + } + connectedDeviceIndex++; } } - // Open Subordinate device. - { - auto device = std::make_shared(); - if (device->open(serialSubordinate)) - { - kinectSettings.wiredSyncMode = K4A_WIRED_SYNC_MODE_SUBORDINATE; - //kinectSettings.subordinateDelayUsec = 100; - device->startCameras(kinectSettings); + int masterDeviceIndex = syncOutDeviceIndex >= 0 ? syncOutDeviceIndex : syncInOutDeviceIndex; - this->kinectDevices.push_back(device); + if (masterDeviceIndex < 0) { + ofLogWarning() << "No master device is detected. Start streams as standalone mode."; + for (auto& device : this->kinectDevices) { + device->startCameras(kinectSettings); } + } else { + sync.setMasterDevice(kinectDevices[masterDeviceIndex].get()); + + // Open Subordinate devices first + for (int i = 0; i < this->kinectDevices.size(); ++i) { + this->kinectDevices[i]->setExposureTimeAbsolute(8000); + if (i != masterDeviceIndex) { + cerr << "sub device : " << this->kinectDevices[i]->getSerialNumber() << endl; + kinectSettings.wiredSyncMode = K4A_WIRED_SYNC_MODE_SUBORDINATE; + kinectSettings.subordinateDelayUsec += 160; + sync.addSubordinateDevice(kinectDevices[i].get()); + + this->kinectDevices[i]->startCameras(kinectSettings); + } + } + + // Open Master device + cerr << "master device : " << this->kinectDevices[masterDeviceIndex]->getSerialNumber() << endl; + kinectSettings.wiredSyncMode = K4A_WIRED_SYNC_MODE_MASTER; + kinectSettings.subordinateDelayUsec = 0; + this->kinectDevices[masterDeviceIndex]->startCameras(kinectSettings); + + sync.start(); } } //-------------------------------------------------------------- void ofApp::exit() { + sync.stop(); for (auto device : this->kinectDevices) { device->close(); @@ -118,6 +144,14 @@ void ofApp::draw() ofDrawBitmapStringHighlight(ofToString(this->fpsCounters[i].getFps(), 2) + " FPS", x + 10, 350, device->isFrameNew() ? ofColor::red : ofColor::black); + if (device->isSyncInConnected()) { + ofDrawBitmapStringHighlight("SyncIn", x + 100, 350, ofColor::blue); + } + if (device->isSyncOutConnected()) { + ofDrawBitmapStringHighlight("SyncOut", x + 200, 350, ofColor::green); + } + ofDrawBitmapStringHighlight("DeviceTime : " + ofToString(device->getColorTexDeviceTime().count()), x + 10, 40); + x += 640; } } diff --git a/example-multi/src/ofApp.h b/example-multi/src/ofApp.h index 0d001e1..1dbcd4c 100644 --- a/example-multi/src/ofApp.h +++ b/example-multi/src/ofApp.h @@ -31,5 +31,6 @@ class ofApp private: std::vector> kinectDevices; + ofxAzureKinect::MultiDeviceSyncCapture sync; std::vector fpsCounters; }; diff --git a/example-playback/addons.make b/example-playback/addons.make new file mode 100644 index 0000000..b155bef --- /dev/null +++ b/example-playback/addons.make @@ -0,0 +1 @@ +ofxAzureKinect diff --git a/example-playback/bin/data/.gitkeep b/example-playback/bin/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/example-playback/example-playback.vcxproj.filters b/example-playback/example-playback.vcxproj.filters new file mode 100644 index 0000000..d599c86 --- /dev/null +++ b/example-playback/example-playback.vcxproj.filters @@ -0,0 +1,87 @@ + + + + + src + + + src + + + src + + + src + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + + + {d8376475-7454-4a24-b08a-aac121d3ad6f} + + + {71834F65-F3A9-211E-73B8-DC85} + + + {D1CD0095-639A-AA0C-2695-A77A} + + + {1D958AB7-749A-B5D4-4289-C982} + + + {5A177E02-2822-7F8A-3873-684D} + + + {94F0548D-4449-5E5B-562B-E676} + + + {3ECE9E8D-EE7A-167B-D044-A335} + + + {72187D30-5AC2-D6C7-38B3-F51C} + + + + + src + + + src + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src + + + addons\ofxAzureKinect\libs\turbojpeg\include + + + + + + diff --git a/example-playback/example-playback.vcxproj.user b/example-playback/example-playback.vcxproj.user new file mode 100644 index 0000000..09a332b --- /dev/null +++ b/example-playback/example-playback.vcxproj.user @@ -0,0 +1,19 @@ + + + + $(ProjectDir)/bin + WindowsLocalDebugger + + + $(ProjectDir)/bin + WindowsLocalDebugger + + + $(ProjectDir)/bin + WindowsLocalDebugger + + + $(ProjectDir)/bin + WindowsLocalDebugger + + \ No newline at end of file diff --git a/example-playback/icon.rc b/example-playback/icon.rc new file mode 100644 index 0000000..7e26eb3 --- /dev/null +++ b/example-playback/icon.rc @@ -0,0 +1,8 @@ +// Icon Resource Definition +#define MAIN_ICON 102 + +#if defined(_DEBUG) +MAIN_ICON ICON "icon_debug.ico" +#else +MAIN_ICON ICON "icon.ico" +#endif diff --git a/example-playback/src/main.cpp b/example-playback/src/main.cpp new file mode 100644 index 0000000..e57370b --- /dev/null +++ b/example-playback/src/main.cpp @@ -0,0 +1,13 @@ +#include "ofMain.h" +#include "ofApp.h" + +//======================================================================== +int main( ){ + ofSetupOpenGL(1024,768,OF_WINDOW); // <-------- setup the GL context + + // this kicks off the running of my app + // can be OF_WINDOW or OF_FULLSCREEN + // pass in width and height too: + ofRunApp(new ofApp()); + +} diff --git a/example-playback/src/ofApp.cpp b/example-playback/src/ofApp.cpp new file mode 100644 index 0000000..c0ca254 --- /dev/null +++ b/example-playback/src/ofApp.cpp @@ -0,0 +1,129 @@ +#include "ofApp.h" + +//-------------------------------------------------------------- +void ofApp::setup() +{ + // place a recording from k4arecorder or example-record in the data folder + filename = ofToDataPath("output.mkv"); + + device.load(filename); + device.startCameras(); +} + +//-------------------------------------------------------------- +void ofApp::update() +{ +} + +//-------------------------------------------------------------- +void ofApp::draw() +{ + ofBackground(128); + + // visualize kinect streams + if (this->device.isStreaming()) + { + this->device.getColorTex().draw(0, 0, 1280, 720); + this->device.getDepthTex().draw(1280, 0, 360, 360); + this->device.getIrTex().draw(1280, 360, 360, 360); + } + + ofDrawBitmapStringHighlight(ofToString(ofGetFrameRate(), 2) + " FPS", 10, 20); + + stringstream ss; + ss << "Press SPACEBAR to play / pause the recording." << endl; + ss << "Press + / - to seek when the recording is paused." << endl; + ss << "Press 'f' to toggle fullscreen."; + ofDrawBitmapStringHighlight(ss.str(), 10, ofGetHeight() - 50); +} + +//-------------------------------------------------------------- +void ofApp::exit() +{ + device.close(); +} + +//-------------------------------------------------------------- +void ofApp::keyPressed(int key) +{ + switch (key) + { + case ' ': + { + play = !play; + if (play) + { + device.play = true; + } + else + { + device.pause = false; + } + break; + } + case '+': + play_head = ofClamp(play_head + .01, 0, 1); + device.seek = play_head; + break; + case '-': + play_head = ofClamp(play_head - .01, 0, 1); + device.seek = play_head; + break; + case 'f': + case 'F': + ofToggleFullscreen(); + break; + default: + break; + } +} + +//-------------------------------------------------------------- +void ofApp::keyReleased(int key) +{ +} + +//-------------------------------------------------------------- +void ofApp::mouseMoved(int x, int y) +{ +} + +//-------------------------------------------------------------- +void ofApp::mouseDragged(int x, int y, int button) +{ +} + +//-------------------------------------------------------------- +void ofApp::mousePressed(int x, int y, int button) +{ +} + +//-------------------------------------------------------------- +void ofApp::mouseReleased(int x, int y, int button) +{ +} + +//-------------------------------------------------------------- +void ofApp::mouseEntered(int x, int y) +{ +} + +//-------------------------------------------------------------- +void ofApp::mouseExited(int x, int y) +{ +} + +//-------------------------------------------------------------- +void ofApp::windowResized(int w, int h) +{ +} + +//-------------------------------------------------------------- +void ofApp::gotMessage(ofMessage msg) +{ +} + +//-------------------------------------------------------------- +void ofApp::dragEvent(ofDragInfo dragInfo) +{ +} diff --git a/example-playback/src/ofApp.h b/example-playback/src/ofApp.h new file mode 100644 index 0000000..e3e6f8b --- /dev/null +++ b/example-playback/src/ofApp.h @@ -0,0 +1,33 @@ +#pragma once + +#include "ofMain.h" + +#include "ofxAzureKinect.h" +#include "ofxAzureKinect/Device.h" + +class ofApp : public ofBaseApp{ + + public: + void setup(); + void update(); + void draw(); + void exit(); + + void keyPressed(int key); + void keyReleased(int key); + void mouseMoved(int x, int y ); + void mouseDragged(int x, int y, int button); + void mousePressed(int x, int y, int button); + void mouseReleased(int x, int y, int button); + void mouseEntered(int x, int y); + void mouseExited(int x, int y); + void windowResized(int w, int h); + void dragEvent(ofDragInfo dragInfo); + void gotMessage(ofMessage msg); + + ofxAzureKinect::Device device; + + string filename; + bool play = false; + float play_head = 0; +}; diff --git a/example-pointcloud/example-pointcloud.vcxproj b/example-pointcloud/example-pointcloud.vcxproj index d77f822..ed3fad5 100644 --- a/example-pointcloud/example-pointcloud.vcxproj +++ b/example-pointcloud/example-pointcloud.vcxproj @@ -100,7 +100,7 @@ %(PreprocessorDefinitions) MultiThreadedDebugDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp $(IntDir) @@ -108,8 +108,8 @@ true Console false - %(AdditionalDependencies);k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -120,7 +120,7 @@ %(PreprocessorDefinitions) MultiThreadedDebugDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp true $(IntDir) @@ -129,8 +129,8 @@ true Console false - %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -140,7 +140,7 @@ %(PreprocessorDefinitions) MultiThreadedDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp true $(IntDir) @@ -152,8 +152,8 @@ true true false - %(AdditionalDependencies);k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -163,7 +163,7 @@ %(PreprocessorDefinitions) MultiThreadedDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp $(IntDir) @@ -174,19 +174,25 @@ true true false - %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + + + + + + diff --git a/example-pointcloud/example-pointcloud.vcxproj.filters b/example-pointcloud/example-pointcloud.vcxproj.filters index df9faff..d599c86 100644 --- a/example-pointcloud/example-pointcloud.vcxproj.filters +++ b/example-pointcloud/example-pointcloud.vcxproj.filters @@ -13,9 +13,18 @@ src + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + @@ -50,9 +59,18 @@ src + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect diff --git a/example-pointcloud/src/ofApp.cpp b/example-pointcloud/src/ofApp.cpp index 1a13117..3ae8bab 100644 --- a/example-pointcloud/src/ofApp.cpp +++ b/example-pointcloud/src/ofApp.cpp @@ -39,7 +39,7 @@ void ofApp::draw() { this->cam.begin(); { - ofScale(0.001f); + //ofScale(0.001f); ofDrawAxis(1000.0f); diff --git a/example-record/addons.make b/example-record/addons.make new file mode 100644 index 0000000..b155bef --- /dev/null +++ b/example-record/addons.make @@ -0,0 +1 @@ +ofxAzureKinect diff --git a/example-record/bin/data/.gitkeep b/example-record/bin/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/example-record/example-record.vcxproj.filters b/example-record/example-record.vcxproj.filters new file mode 100644 index 0000000..d599c86 --- /dev/null +++ b/example-record/example-record.vcxproj.filters @@ -0,0 +1,87 @@ + + + + + src + + + src + + + src + + + src + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + + + {d8376475-7454-4a24-b08a-aac121d3ad6f} + + + {71834F65-F3A9-211E-73B8-DC85} + + + {D1CD0095-639A-AA0C-2695-A77A} + + + {1D958AB7-749A-B5D4-4289-C982} + + + {5A177E02-2822-7F8A-3873-684D} + + + {94F0548D-4449-5E5B-562B-E676} + + + {3ECE9E8D-EE7A-167B-D044-A335} + + + {72187D30-5AC2-D6C7-38B3-F51C} + + + + + src + + + src + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src + + + addons\ofxAzureKinect\libs\turbojpeg\include + + + + + + diff --git a/example-record/example-record.vcxproj.user b/example-record/example-record.vcxproj.user new file mode 100644 index 0000000..09a332b --- /dev/null +++ b/example-record/example-record.vcxproj.user @@ -0,0 +1,19 @@ + + + + $(ProjectDir)/bin + WindowsLocalDebugger + + + $(ProjectDir)/bin + WindowsLocalDebugger + + + $(ProjectDir)/bin + WindowsLocalDebugger + + + $(ProjectDir)/bin + WindowsLocalDebugger + + \ No newline at end of file diff --git a/example-record/icon.rc b/example-record/icon.rc new file mode 100644 index 0000000..7e26eb3 --- /dev/null +++ b/example-record/icon.rc @@ -0,0 +1,8 @@ +// Icon Resource Definition +#define MAIN_ICON 102 + +#if defined(_DEBUG) +MAIN_ICON ICON "icon_debug.ico" +#else +MAIN_ICON ICON "icon.ico" +#endif diff --git a/example-record/src/main.cpp b/example-record/src/main.cpp new file mode 100644 index 0000000..c00ea15 --- /dev/null +++ b/example-record/src/main.cpp @@ -0,0 +1,13 @@ +#include "ofMain.h" +#include "ofApp.h" + +//======================================================================== +int main( ){ + ofSetupOpenGL(2048,1536,OF_WINDOW); // <-------- setup the GL context + + // this kicks off the running of my app + // can be OF_WINDOW or OF_FULLSCREEN + // pass in width and height too: + ofRunApp(new ofApp()); + +} diff --git a/example-record/src/ofApp.cpp b/example-record/src/ofApp.cpp new file mode 100644 index 0000000..63922ef --- /dev/null +++ b/example-record/src/ofApp.cpp @@ -0,0 +1,199 @@ +#include "ofApp.h" + +//-------------------------------------------------------------- +void ofApp::setup() +{ + ofSetVerticalSync(false); + ofLogNotice(__FUNCTION__) << "Found " << ofxAzureKinect::Device::getInstalledCount() << " installed devices."; + + auto settings = ofxAzureKinect::DeviceSettings(); + settings.colorResolution = K4A_COLOR_RESOLUTION_1536P; + settings.depthMode = K4A_DEPTH_MODE_NFOV_UNBINNED; + settings.colorFormat = K4A_IMAGE_FORMAT_COLOR_MJPG; // BRGA32 not supported for recording + settings.cameraFps = K4A_FRAMES_PER_SECOND_30; + settings.syncImages = true; + settings.updateWorld = false; + settings.enableIMU = true; + settings.updateVbo = false; + + if (this->sensor.open()) + { + this->sensor.startCameras(settings); + this->sensor.setExposureTimeAbsolute(10000); + } +} + +//-------------------------------------------------------------- +void ofApp::exit() +{ + this->sensor.close(); +} + +//-------------------------------------------------------------- +void ofApp::update() +{ + + // If we are recording, update the total recording time + if (sensor.isRecording() && ofGetElapsedTimef() > recording_start) + recording_duration = ofGetElapsedTimef() - recording_start; + + if (sensor.isFrameNew()) { + fpsCounter.newFrame(); + } + +} + +//-------------------------------------------------------------- +void ofApp::draw() +{ + ofBackground(128); + + // visualize kinect streams + if (this->sensor.isStreaming()) + { + this->sensor.getColorTex().draw(0, 0, 2048, 1536); + this->sensor.getDepthTex().draw(0, 0, 360, 360); + this->sensor.getIrTex().draw(0, 360, 360, 360); + } + + if (sensor.isRecording()) + { + draw_recording_animation(); + } + + ofDrawBitmapStringHighlight("Render : " + ofToString(ofGetFrameRate(), 2) + " FPS", 10, 20); + ofDrawBitmapStringHighlight("Camera : " + ofToString(this->fpsCounter.getFps(), 2) + " FPS", 10, 40); + + stringstream ss; + ss << "Press SPACEBAR to start/stop a recording." << endl; + ss << "\tRecordings are saved to the bin/data/ directory." << endl; + ss << "Press 'f' to toggle fullscreen."; + ofDrawBitmapStringHighlight(ss.str(), 10, ofGetHeight() - 50); +} + +//-------------------------------------------------------------- +void ofApp::draw_recording_animation() +{ + + float a = ofMap(sin(ofGetElapsedTimef() * 3.5), -1, 1, 0, 120); + ofColor col; + string msg; + + float recording_countdown = sensor.getRecordingTimerDelay(); + if (recording_countdown > 0) + { + msg = "Starting Recording In: " + ofToString(recording_countdown, 2); + col = ofColor(120, a); + } + if (recording_duration > 0) + { + msg = "Recording Time: " + ofToString(recording_duration, 2); + col = ofColor(255, 0, 0, a); + } + + ofPushStyle(); + ofSetColor(col); + ofDrawRectangle(0, 0, ofGetWidth(), ofGetHeight()); + ofPopStyle(); + + ofDrawBitmapStringHighlight(msg, 10, 40); +} + +//-------------------------------------------------------------- +void ofApp::keyPressed(int key) +{ + switch (key) + { + case ' ': + { + if (!sensor.isRecording()) { + sensor.startRecording(); + } + else { + sensor.stopRecording(); + } + if (sensor.isRecording()) + { + recording_delay = sensor.getRecordingTimerDelay(); + recording_start = ofGetElapsedTimef() + recording_delay; + } + else + { + recording_delay = 0; + recording_start = 0; + recording_duration = 0; + } + break; + } + case 'f': + case 'F': + ofToggleFullscreen(); + break; + case OF_KEY_UP: + { + auto exposure = this->sensor.getExposureTimeAbsolute(); + cerr << "exposure : " << exposure << endl; + this->sensor.setExposureTimeAbsolute(exposure + 1); + break; + } + case OF_KEY_DOWN: + { + auto exposure = this->sensor.getExposureTimeAbsolute(); + cerr << "exposure : " << exposure << endl; + this->sensor.setExposureTimeAbsolute(exposure / 3); + break; + } + default: + break; + } +} + +//-------------------------------------------------------------- +void ofApp::keyReleased(int key) +{ +} + +//-------------------------------------------------------------- +void ofApp::mouseMoved(int x, int y) +{ +} + +//-------------------------------------------------------------- +void ofApp::mouseDragged(int x, int y, int button) +{ +} + +//-------------------------------------------------------------- +void ofApp::mousePressed(int x, int y, int button) +{ +} + +//-------------------------------------------------------------- +void ofApp::mouseReleased(int x, int y, int button) +{ +} + +//-------------------------------------------------------------- +void ofApp::mouseEntered(int x, int y) +{ +} + +//-------------------------------------------------------------- +void ofApp::mouseExited(int x, int y) +{ +} + +//-------------------------------------------------------------- +void ofApp::windowResized(int w, int h) +{ +} + +//-------------------------------------------------------------- +void ofApp::gotMessage(ofMessage msg) +{ +} + +//-------------------------------------------------------------- +void ofApp::dragEvent(ofDragInfo dragInfo) +{ +} diff --git a/example-record/src/ofApp.h b/example-record/src/ofApp.h new file mode 100644 index 0000000..11ff419 --- /dev/null +++ b/example-record/src/ofApp.h @@ -0,0 +1,35 @@ +#pragma once + +#include "ofMain.h" +#include "ofxAzureKinect.h" + +class ofApp : public ofBaseApp{ + + public: + void setup(); + void exit(); + void update(); + void draw(); + + void keyPressed(int key); + void keyReleased(int key); + void mouseMoved(int x, int y ); + void mouseDragged(int x, int y, int button); + void mousePressed(int x, int y, int button); + void mouseReleased(int x, int y, int button); + void mouseEntered(int x, int y); + void mouseExited(int x, int y); + void windowResized(int w, int h); + void dragEvent(ofDragInfo dragInfo); + void gotMessage(ofMessage msg); + + ofxAzureKinect::Device sensor; + ofFpsCounter fpsCounter; + + void draw_recording_animation(); + + float recording_delay=0; + float recording_start=0; + float recording_duration=0; + +}; diff --git a/example-shader/example-shader.vcxproj b/example-shader/example-shader.vcxproj index 779235d..ef5e4f2 100644 --- a/example-shader/example-shader.vcxproj +++ b/example-shader/example-shader.vcxproj @@ -100,7 +100,7 @@ %(PreprocessorDefinitions) MultiThreadedDebugDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp $(IntDir) @@ -108,8 +108,8 @@ true Console false - %(AdditionalDependencies);k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -120,7 +120,7 @@ %(PreprocessorDefinitions) MultiThreadedDebugDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp true $(IntDir) @@ -129,8 +129,8 @@ true Console false - %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -140,7 +140,7 @@ %(PreprocessorDefinitions) MultiThreadedDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp true $(IntDir) @@ -152,8 +152,8 @@ true true false - %(AdditionalDependencies);k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -163,7 +163,7 @@ %(PreprocessorDefinitions) MultiThreadedDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp $(IntDir) @@ -174,19 +174,25 @@ true true false - %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + + + + + + diff --git a/example-shader/example-shader.vcxproj.filters b/example-shader/example-shader.vcxproj.filters index df9faff..d599c86 100644 --- a/example-shader/example-shader.vcxproj.filters +++ b/example-shader/example-shader.vcxproj.filters @@ -13,9 +13,18 @@ src + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + @@ -50,9 +59,18 @@ src + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect diff --git a/example-streams/example-streams.vcxproj b/example-streams/example-streams.vcxproj index b5d8997..eda7b87 100644 --- a/example-streams/example-streams.vcxproj +++ b/example-streams/example-streams.vcxproj @@ -100,7 +100,7 @@ %(PreprocessorDefinitions) MultiThreadedDebugDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp $(IntDir) @@ -108,8 +108,8 @@ true Console false - %(AdditionalDependencies);k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -120,7 +120,7 @@ %(PreprocessorDefinitions) MultiThreadedDebugDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp true $(IntDir) @@ -129,8 +129,8 @@ true Console false - %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -140,7 +140,7 @@ %(PreprocessorDefinitions) MultiThreadedDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp true $(IntDir) @@ -152,8 +152,8 @@ true true false - %(AdditionalDependencies);k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib @@ -163,7 +163,7 @@ %(PreprocessorDefinitions) MultiThreadedDLL Level3 - %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include + %(AdditionalIncludeDirectories);src;..\..\..\addons\ofxAzureKinect\libs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\include;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Debug;..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;..\..\..\addons\ofxAzureKinect\src;..\..\..\addons\ofxAzureKinect\src\ofxAzureKinect;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\include;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\include CompileAsCpp $(IntDir) @@ -174,19 +174,25 @@ true true false - %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4abt.lib - %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;C:\Program Files\Azure Kinect SDK v1.4.0\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + %(AdditionalDependencies);turbojpeg-static.lib;k4a.lib;k4arecord.lib;k4abt.lib + %(AdditionalLibraryDirectories);..\..\..\addons\ofxAzureKinect\libs\turbojpeg\lib\vs\x64\Release;C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib;C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\lib + + + + + + diff --git a/example-streams/example-streams.vcxproj.filters b/example-streams/example-streams.vcxproj.filters index df9faff..d599c86 100644 --- a/example-streams/example-streams.vcxproj.filters +++ b/example-streams/example-streams.vcxproj.filters @@ -13,9 +13,18 @@ src + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + @@ -50,9 +59,18 @@ src + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect + + addons\ofxAzureKinect\src\ofxAzureKinect + + + addons\ofxAzureKinect\src\ofxAzureKinect + addons\ofxAzureKinect\src\ofxAzureKinect diff --git a/example-streams/src/ofApp.cpp b/example-streams/src/ofApp.cpp index 8682e51..4b57adf 100644 --- a/example-streams/src/ofApp.cpp +++ b/example-streams/src/ofApp.cpp @@ -10,6 +10,7 @@ void ofApp::setup() if (this->kinectDevice.open()) { auto kinectSettings = ofxAzureKinect::DeviceSettings(); + kinectSettings.colorResolution = K4A_COLOR_RESOLUTION_1080P; kinectSettings.syncImages = false; kinectSettings.updateWorld = false; this->kinectDevice.startCameras(kinectSettings); diff --git a/src/ofxAzureKinect/BodyTracker.cpp b/src/ofxAzureKinect/BodyTracker.cpp new file mode 100644 index 0000000..7491f22 --- /dev/null +++ b/src/ofxAzureKinect/BodyTracker.cpp @@ -0,0 +1,315 @@ +#include "BodyTracker.h" + +namespace ofxAzureKinect +{ + + BodyTracker::BodyTracker(k4a_calibration_t calibration, k4abt_tracker_configuration_t config) + { + k4abt_tracker_create(&calibration, config, &tracker); + active = true; + load_shaders(); + } + + void BodyTracker::update(k4a_capture_t capture) + { + k4a_wait_result_t enqueueResult = k4abt_tracker_enqueue_capture(tracker, capture, K4A_WAIT_INFINITE); + if (enqueueResult == K4A_WAIT_RESULT_FAILED) + { + ofLogError(__FUNCTION__) << "Failed adding capture to tracker process queue!"; + } + else + { + k4abt_frame_t bodyFrame = nullptr; + k4a_wait_result_t result = k4abt_tracker_pop_result(tracker, &bodyFrame, K4A_WAIT_INFINITE); + if (result == K4A_WAIT_RESULT_SUCCEEDED) + { + // Probe for a body index map image. + k4a::image bodyIndexImg = k4abt_frame_get_body_index_map(bodyFrame); + + // Update the Body Index Map ofPixels + const auto bodyIndexSize = glm::ivec2(bodyIndexImg.get_width_pixels(), bodyIndexImg.get_height_pixels()); + if (!this->bodyIndexPix.isAllocated()) + { + this->bodyIndexPix.allocate(bodyIndexSize.x, bodyIndexSize.y, 1); + } + + // Set the ofPixels + const auto bodyIndexData = reinterpret_cast(bodyIndexImg.get_buffer()); + this->bodyIndexPix.setFromPixels(bodyIndexData, bodyIndexSize.x, bodyIndexSize.y, 1); + + // Release the body index map image + ofLogVerbose(__FUNCTION__) << "Capture BodyIndex " << bodyIndexSize.x << "x" << bodyIndexSize.y << " stride: " << bodyIndexImg.get_stride_bytes() << "."; + bodyIndexImg.reset(); + + // Set number of bodies found + num_bodies = k4abt_frame_get_num_bodies(bodyFrame); + ofLogVerbose(__FUNCTION__) << num_bodies << " bodies found!"; + + // Update Found Skeletons + this->skeletons.resize(num_bodies); + this->bodyIDs.resize(num_bodies); + skeleton_meshes.clear(); + for (size_t i = 0; i < num_bodies; i++) + { + k4abt_skeleton_t skeleton; + k4abt_frame_get_body_skeleton(bodyFrame, i, &skeleton); + this->skeletons[i] = skeleton; + uint32_t id = k4abt_frame_get_body_id(bodyFrame, i); + this->bodyIDs[i] = id; + + skeleton_meshes.push_back(new ofVboMesh()); + skeleton_meshes[skeleton_meshes.size() - 1]->setMode(OF_PRIMITIVE_LINES); + } + + // Release body frame once we're finished. + k4abt_frame_release(bodyFrame); + } + } + } + + void BodyTracker::update_texture() + { + if (!this->bodyIndexTex.isAllocated()) + { + this->bodyIndexTex.allocate(this->bodyIndexPix); + this->bodyIndexTex.setTextureMinMagFilter(GL_NEAREST, GL_NEAREST); + } + this->bodyIndexTex.loadData(this->bodyIndexPix); + } + + void BodyTracker::set_joint_smoothing(float val) + { + joint_smoothing = val; + k4abt_tracker_set_temporal_smoothing(tracker, joint_smoothing); + } + + float BodyTracker::get_joint_smoothing() + { + return joint_smoothing; + } + + // The value for each pixel represents which body the pixel belongs to. + // It can be either background (K4ABT_BODY_INDEX_MAP_BACKGROUND) + // or the index of a detected k4abt_body_t. + // https://docs.microsoft.com/en-us/azure/kinect-dk/body-index-map + void BodyTracker::draw_body_map(int x, int y, int w, int h) + { + // Draw the 2D Map of all Detected Bodies + bodyIndexTex.draw(x, y, w, h); + } + + void BodyTracker::draw_point_clouds(ofTexture depth_texture, ofTexture depth_to_world_texture) + { + ofEnableDepthTest(); + constexpr int kMaxBodies = 6; + int _bodyIDs[kMaxBodies]; + int i = 0; + while (i < num_bodies) + { + _bodyIDs[i] = bodyIDs[i]; + ++i; + } + while (i < kMaxBodies) + { + _bodyIDs[i] = 0; + ++i; + } + + this->shader.begin(); + { + this->shader.setUniformTexture("uDepthTex", depth_texture, 1); + this->shader.setUniformTexture("uBodyIndexTex", bodyIndexTex, 2); + this->shader.setUniformTexture("uWorldTex", depth_to_world_texture, 3); + this->shader.setUniform2i("uFrameSize", depth_texture.getWidth(), depth_texture.getHeight()); + this->shader.setUniform1iv("uBodyIDs", _bodyIDs, kMaxBodies); + + int numPoints = depth_texture.getWidth() * depth_texture.getHeight(); + this->pointsVbo.drawInstanced(GL_POINTS, 0, 1, numPoints); + } + this->shader.end(); + ofDisableDepthTest(); + } + + void BodyTracker::draw_skeletons() + { + for (int i = 0; i < num_bodies; i++) + { + draw_skeleton(i); + } + } + + void BodyTracker::draw_skeleton(int id) + { + if (id >= num_bodies) + { + ofLogError(__FUNCTION__) << "Invaild Body Index {" << id << "}. Body does not exist."; + return; + } + ofPushStyle(); + + auto skeleton = skeletons[id]; + // Draw Joints + for (int i = 0; i < K4ABT_JOINT_COUNT; ++i) + { + auto joint = skeleton.joints[i]; + ofPushMatrix(); + { + glm::mat4 transform = glm::translate(toGlm(joint.position)) * glm::toMat4(toGlm(joint.orientation)); + ofMultMatrix(transform); + ofNoFill(); + ofDrawBox(15); + ofDrawAxis(75.0f); + + if (i == K4ABT_JOINT_SPINE_CHEST) + { + ofPushMatrix(); + ofTranslate(500, 0, 0); + ofDrawBitmapStringHighlight(ofToString(bodyIDs[id]), 0, 0); + ofPopMatrix(); + } + } + ofPopMatrix(); + } + + // Draw Bones + auto &vertices = skeleton_meshes[id]->getVertices(); + vertices.resize(50); + + int vdx = 0; + + // Spine. + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_PELVIS].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SPINE_NAVEL].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SPINE_NAVEL].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SPINE_CHEST].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SPINE_CHEST].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NECK].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NECK].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_HEAD].position); + + // Head. + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_HEAD].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NOSE].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NOSE].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_EYE_LEFT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_EYE_LEFT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_EAR_LEFT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NOSE].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_EYE_RIGHT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_EYE_RIGHT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_EAR_RIGHT].position); + + // Left Leg. + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_PELVIS].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_HIP_LEFT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_HIP_LEFT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_KNEE_LEFT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_KNEE_LEFT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ANKLE_LEFT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ANKLE_LEFT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_FOOT_LEFT].position); + + // Right leg. + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_PELVIS].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_HIP_RIGHT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_HIP_RIGHT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_KNEE_RIGHT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_KNEE_RIGHT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ANKLE_RIGHT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ANKLE_RIGHT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_FOOT_RIGHT].position); + + // Left arm. + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NECK].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_CLAVICLE_LEFT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_CLAVICLE_LEFT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SHOULDER_LEFT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SHOULDER_LEFT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ELBOW_LEFT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ELBOW_LEFT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_WRIST_LEFT].position); + + // Right arm. + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_NECK].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_CLAVICLE_RIGHT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_CLAVICLE_RIGHT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SHOULDER_RIGHT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_SHOULDER_RIGHT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ELBOW_RIGHT].position); + + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_ELBOW_RIGHT].position); + vertices[vdx++] = toGlm(skeleton.joints[K4ABT_JOINT_WRIST_RIGHT].position); + + ofSetColor(200); + skeleton_meshes[id]->draw(); + + ofPopStyle(); + } + + //-------------------------------------------------------------- + void BodyTracker::load_shaders() + { + auto shaderSettings = ofShaderSettings(); + shaderSettings.shaderFiles[GL_VERTEX_SHADER] = "shaders/render.vert"; + shaderSettings.shaderFiles[GL_FRAGMENT_SHADER] = "shaders/render.frag"; + shaderSettings.intDefines["BODY_INDEX_MAP_BACKGROUND"] = K4ABT_BODY_INDEX_MAP_BACKGROUND; + shaderSettings.bindDefaults = true; + + if (this->shader.setup(shaderSettings)) + { + ofLogNotice(__FUNCTION__) << "Success loading shader!"; + } + else + { + ofLogError(__FUNCTION__) << "Could not load shader. Check that they are in the /bin/data/shader directory."; + } + + // Setup PointCloud VBO + std::vector verts(1); + this->pointsVbo.setVertexData(verts.data(), verts.size(), GL_STATIC_DRAW); + } + + size_t BodyTracker::get_num_bodies() const + { + return num_bodies; + } + + const vector &BodyTracker::get_skeletons() const + { + return skeletons; + } + + const k4abt_skeleton_t &BodyTracker::get_skeleton(int i) const + { + return skeletons[i]; + } + + const ofPixels &BodyTracker::get_body_index_map_pix() const + { + return bodyIndexPix; + } + + const ofTexture &BodyTracker::get_body_index_map_tex() const + { + return bodyIndexTex; + } + +} // namespace ofxAzureKinect \ No newline at end of file diff --git a/src/ofxAzureKinect/BodyTracker.h b/src/ofxAzureKinect/BodyTracker.h new file mode 100644 index 0000000..ae210fe --- /dev/null +++ b/src/ofxAzureKinect/BodyTracker.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include + +#include "ofMain.h" + +#include "Types.h" + +namespace ofxAzureKinect +{ + class BodyTracker + { + + public: + BodyTracker(){}; + BodyTracker(k4a_calibration_t calibration, k4abt_tracker_configuration_t config); + ~BodyTracker(){}; + + void update(k4a_capture_t capture); + void update_texture(); // why do I get a seg fault when called in update?? + + const vector &get_skeletons() const; + const k4abt_skeleton_t &get_skeleton(int id) const; + const vector &get_body_ids() const; + + size_t get_num_bodies() const; + + const ofPixels &get_body_index_map_pix() const; + const ofTexture &get_body_index_map_tex() const; + + void set_joint_smoothing(float val); + float get_joint_smoothing(); + + bool is_active() { return active; } + + void draw_body_map(int x = 0, int y = 0, int w = 360, int h = 360); + void draw_point_clouds(ofTexture depth_texture, ofTexture depth_to_world_texture); + void draw_skeletons(); + void draw_skeleton(int id); + + private: + k4abt_tracker_t tracker; + + size_t num_bodies = 0; + + bool active = false; + + std::vector skeletons; + std::vector bodyIDs; + + // Visualization + ofPixels bodyIndexPix; + ofTexture bodyIndexTex; + + void load_shaders(); + ofShader shader; + vector skeleton_meshes; + ofVbo pointsVbo; + + ofParameter joint_smoothing{"Joint Smoothing", 0.0f, 0.0f, 1.0f}; + }; +} // namespace ofxAzureKinect \ No newline at end of file diff --git a/src/ofxAzureKinect/Device.cpp b/src/ofxAzureKinect/Device.cpp index b5161fe..25cd82c 100644 --- a/src/ofxAzureKinect/Device.cpp +++ b/src/ofxAzureKinect/Device.cpp @@ -2,10 +2,27 @@ #include "ofLog.h" -const int32_t TIMEOUT_IN_MS = 1000; namespace ofxAzureKinect { + static constexpr int32_t TIMEOUT_IN_MS = 1000; + static constexpr int64_t WAIT_FOR_SYNCHRONIZED_CAPTURE_TIMEOUT = 60000; + + static void log_lagging_time(const char *lagger, k4a::capture &master, k4a::capture &sub) + { + int32_t diff = master.get_color_image().get_device_timestamp().count() - sub.get_color_image().get_device_timestamp().count(); + std::cout << std::setw(6) << lagger << " lagging: mc:" << std::setw(6) + << master.get_color_image().get_device_timestamp().count() << "us sc:" << std::setw(6) + << sub.get_color_image().get_device_timestamp().count() << "us diff:" << diff << endl; + } + + static void log_synced_image_time(k4a::capture &master, k4a::capture &sub) + { + int32_t diff = master.get_color_image().get_device_timestamp().count() - sub.get_color_image().get_device_timestamp().count(); + std::cout << "Sync'd capture: mc:" << std::setw(6) << master.get_color_image().get_device_timestamp().count() + << "us sc:" << std::setw(6) << sub.get_color_image().get_device_timestamp().count() << "us diff:" << diff << endl; + } + DeviceSettings::DeviceSettings(int idx) : depthMode(K4A_DEPTH_MODE_WFOV_2X2BINNED) , colorResolution(K4A_COLOR_RESOLUTION_2160P) @@ -19,6 +36,7 @@ namespace ofxAzureKinect , updateWorld(true) , updateVbo(true) , syncImages(true) + , enableIMU(false) {} BodyTrackingSettings::BodyTrackingSettings() @@ -28,6 +46,119 @@ namespace ofxAzureKinect , updateBodies(false) {} + void Device::Frame::swapFrame(Frame & f) + { + this->depthPix.swap(f.depthPix); + std::swap(this->depthPixDeviceTime, f.depthPixDeviceTime); + + if (this->bColorPixUpdated) { + this->colorPix.swap(f.colorPix); + std::swap(this->colorPixDeviceTime, f.colorPixDeviceTime); + std::swap(this->bColorPixUpdated, f.bColorPixUpdated); + } + + this->irPix.swap(f.irPix); + this->depthInColorPix.swap(f.depthInColorPix); + this->colorInDepthPix.swap(f.colorInDepthPix); + this->bodyIndexPix.swap(f.bodyIndexPix); + std::swap(this->bodySkeletons, f.bodySkeletons); + std::swap(this->bodyIDs, f.bodyIDs); + + std::swap(this->positionCache, f.positionCache); + std::swap(this->uvCache, f.uvCache); + std::swap(this->numPoints, f.numPoints); + } + + void Device::Frame::reset() + { + *this = Frame(); + } + + Device::JpegDecodeThread::JpegDecodeThread() : jpegDecompressor(tjInitDecompress()) + { + } + + Device::JpegDecodeThread::~JpegDecodeThread() + { + toProcess.close(); + processed.close(); + tjDestroy(jpegDecompressor); + } + + void Device::JpegDecodeThread::start() + { + if (isThreadRunning()) { + return; + } + startThread(); + } + + void Device::JpegDecodeThread::stop() + { + if (!isThreadRunning()) { + return; + } + waitForThread(); + + // clear remaining tasks + if (!toProcess.empty()) { + JpegTask b; + while (toProcess.tryReceive(b)) {} + } + if (!processed.empty()) { + DecodedPix b; + while (processed.tryReceive(b)) {} + } + } + + bool Device::JpegDecodeThread::pushTaskIfEmpty(JpegTask & b) + { + if (toProcess.empty()) { + toProcess.send(b); + return true; + } + return false; + } + + bool Device::JpegDecodeThread::update(ofPixels & outPix, std::chrono::microseconds & outTime) + { + DecodedPix ret; + bool onceReceived = false; + while (processed.tryReceive(ret)) { + onceReceived = true; + } + + if (onceReceived) { + outPix = std::move(ret.colorPix); + outTime = ret.colorPixDeviceTime; + } + return onceReceived; + } + + void Device::JpegDecodeThread::threadedFunction() + { + while (isThreadRunning()) { + JpegTask b; + if (toProcess.tryReceive(b)) { + DecodedPix pix; + int width, height, jpegSubsamp, jpegColorspace; + const int header = tjDecompressHeader3(this->jpegDecompressor, + (const unsigned char*)b.colorPixBuf.getData(), + static_cast(b.colorPixBuf.size()), + &width, &height, &jpegSubsamp, &jpegColorspace); + pix.colorPix.allocate(width, height, OF_PIXELS_BGRA); + const int decompressStatus = tjDecompress2(this->jpegDecompressor, + (const unsigned char*)b.colorPixBuf.getData(), + static_cast(b.colorPixBuf.size()), + pix.colorPix.getData(), + width, 0, height, TJPF_BGRA, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE); + pix.colorPixDeviceTime = b.colorPixDeviceTime; + processed.send(pix); + } + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + } + int Device::getInstalledCount() { return k4a_device_get_installed_count(); @@ -40,6 +171,7 @@ namespace ofxAzureKinect , bOpen(false) , bStreaming(false) , bNewFrame(false) + , bNewBuffer(false) , bUpdateColor(false) , bUpdateIr(false) , bUpdateBodies(false) @@ -47,6 +179,13 @@ namespace ofxAzureKinect , bUpdateVbo(false) , bodyTracker(nullptr) , jpegDecompressor(tjInitDecompress()) + , bPlayback(false) + , bEnableIMU(false) + , bMultiDeviceSyncCapture(false) + , bRecording(false) + , bAsyncJpegDecode(false) + , bEnableAutoUpdate(true) + , bEnableThread(true) {} Device::~Device() @@ -56,6 +195,54 @@ namespace ofxAzureKinect tjDestroy(jpegDecompressor); } + bool Device::load(string filename) + { + bPlayback = true; + playback = new Playback(); + + if (playback->load(filename)) + { + k4a_record_configuration_t playback_config = playback->getDeviceSettings(); + + this->config = K4A_DEVICE_CONFIG_INIT_DISABLE_ALL; + this->config.depth_mode = playback_config.depth_mode; + this->config.color_format = playback_config.color_format; + this->config.color_resolution = playback_config.color_resolution; + this->config.camera_fps = playback_config.camera_fps; + this->bEnableIMU = playback_config.imu_track_enabled; + + this->config.wired_sync_mode = playback_config.wired_sync_mode; + this->config.depth_delay_off_color_usec = playback_config.depth_delay_off_color_usec; + this->config.subordinate_delay_off_master_usec = playback_config.subordinate_delay_off_master_usec; + + this->serialNumber = playback->getSerialNumber(); + this->bUpdateColor = playback_config.color_track_enabled; + this->bUpdateIr = playback_config.ir_track_enabled; + this->bUpdateWorld = playback_config.depth_track_enabled; + this->bUpdateVbo = playback_config.depth_track_enabled; + + // Add Playback Listeners + this->eventListeners.push(this->play.newListener([this](bool) { + listener_playback_play(this->play); + })); + this->eventListeners.push(this->pause.newListener([this](bool) { + listener_playback_pause(this->pause); + })); + this->eventListeners.push(this->stop.newListener([this](bool) { + listener_playback_stop(this->stop); + })); + this->eventListeners.push(this->seek.newListener([this](bool) { + listener_playback_seek(this->seek); + })); + + ofLogNotice(__FUNCTION__) << "Successfully opened device " << this->index << " with serial number " << this->serialNumber << "."; + + bOpen = true; + return true; + } + return false; + } + bool Device::open(uint32_t idx) { if (this->bOpen) @@ -87,9 +274,10 @@ namespace ofxAzureKinect this->bOpen = true; return true; + //return this->open(DeviceSettings(idx), BodyTrackingSettings()); } - bool Device::open(const std::string& serialNumber) + bool Device::open(const std::string & serialNumber) { if (this->bOpen) { @@ -141,15 +329,25 @@ namespace ofxAzureKinect bool Device::close() { - if (!this->bOpen) return false; + if (!this->bOpen) + return false; + + if (bPlayback) + { + this->stopCameras(); + } + else + { + this->stopCameras(); - this->stopCameras(); + this->device.close(); + } - this->device.close(); + this->eventListeners.unsubscribeAll(); this->index = -1; - this->serialNumber = ""; this->bOpen = false; + this->serialNumber = ""; return true; } @@ -162,39 +360,58 @@ namespace ofxAzureKinect return false; } - // Generate device config. - this->config = K4A_DEVICE_CONFIG_INIT_DISABLE_ALL; - this->config.depth_mode = deviceSettings.depthMode; - this->config.color_format = deviceSettings.colorFormat; - this->config.color_resolution = deviceSettings.colorResolution; - this->config.camera_fps = deviceSettings.cameraFps; - this->config.synchronized_images_only = deviceSettings.syncImages; - - this->config.wired_sync_mode = deviceSettings.wiredSyncMode; - this->config.depth_delay_off_color_usec = deviceSettings.depthDelayUsec; - this->config.subordinate_delay_off_master_usec = deviceSettings.subordinateDelayUsec; - - // Generate tracker config. - this->trackerConfig.sensor_orientation = bodyTrackingSettings.sensorOrientation; - this->trackerConfig.gpu_device_id = bodyTrackingSettings.gpuDeviceID; - - // Set update flags. - this->bUpdateColor = deviceSettings.updateColor; - this->bUpdateIr = deviceSettings.updateIr; - this->bUpdateWorld = deviceSettings.updateWorld; - this->bUpdateVbo = deviceSettings.updateWorld && deviceSettings.updateVbo; - + if (!bPlayback) { + // Generate device config. + this->config = K4A_DEVICE_CONFIG_INIT_DISABLE_ALL; + this->config.depth_mode = deviceSettings.depthMode; + this->config.color_format = deviceSettings.colorFormat; + this->config.color_resolution = deviceSettings.colorResolution; + this->config.camera_fps = deviceSettings.cameraFps; + this->config.synchronized_images_only = deviceSettings.syncImages; + this->bEnableIMU = deviceSettings.enableIMU; + + this->config.wired_sync_mode = deviceSettings.wiredSyncMode; + this->config.depth_delay_off_color_usec = deviceSettings.depthDelayUsec; + this->config.subordinate_delay_off_master_usec = deviceSettings.subordinateDelayUsec; + + // Generate tracker config. + this->trackerConfig.sensor_orientation = bodyTrackingSettings.sensorOrientation; + this->trackerConfig.gpu_device_id = bodyTrackingSettings.gpuDeviceID; + + // Set update flags. + this->bUpdateColor = deviceSettings.updateColor; + this->bUpdateIr = deviceSettings.updateIr; + this->bUpdateWorld = deviceSettings.updateWorld; + this->bUpdateVbo = deviceSettings.updateWorld && deviceSettings.updateVbo; + } this->bUpdateBodies = bodyTrackingSettings.updateBodies; // Get calibration. - try + if (bPlayback) { - this->calibration = this->device.get_calibration(this->config.depth_mode, this->config.color_resolution); + auto calibration_handle = playback->getCalibration(); + this->calibration.depth_camera_calibration = calibration_handle.depth_camera_calibration; + this->calibration.color_camera_calibration = calibration_handle.color_camera_calibration; + this->calibration.depth_mode = calibration_handle.depth_mode; + this->calibration.color_resolution = calibration_handle.color_resolution; + + for (int i = 0; i < 4; i++) + { + for (int j = 0; j < 4; j++) + this->calibration.extrinsics[i][j] = calibration_handle.extrinsics[i][j]; + } } - catch (const k4a::error& e) + else { - ofLogError(__FUNCTION__) << e.what(); - return false; + try + { + this->calibration = this->device.get_calibration(this->config.depth_mode, this->config.color_resolution); + } + catch (const k4a::error &e) + { + ofLogError(__FUNCTION__) << e.what(); + return false; + } } if (this->bUpdateColor) @@ -205,14 +422,8 @@ namespace ofxAzureKinect if (this->bUpdateBodies) { - // Create tracker. - k4abt_tracker_create(&this->calibration, this->trackerConfig, &this->bodyTracker); - - // Add joint smoothing parameter listener. - this->eventListeners.push(this->jointSmoothing.newListener([this](float &) - { - k4abt_tracker_set_temporal_smoothing(this->bodyTracker, this->jointSmoothing); - })); + // Create Body Tracker + tracker = BodyTracker(calibration, trackerConfig); } if (this->bUpdateWorld) @@ -228,35 +439,66 @@ namespace ofxAzureKinect } // Check compatible sync mode and connection. - if (this->config.wired_sync_mode == K4A_WIRED_SYNC_MODE_MASTER && !this->isSyncOutConnected()) - { - ofLogWarning(__FUNCTION__) << "Wired sync mode set to Master but Sync Out not connected! Reverting to Standalone."; - this->config.wired_sync_mode = K4A_WIRED_SYNC_MODE_STANDALONE; - } - else if (this->config.wired_sync_mode == K4A_WIRED_SYNC_MODE_SUBORDINATE && !this->isSyncInConnected()) - { - ofLogWarning(__FUNCTION__) << "Wired sync mode set to Subordinate but Sync In not connected! Reverting to Standalone."; - this->config.wired_sync_mode = K4A_WIRED_SYNC_MODE_STANDALONE; - } + if (!bPlayback) { + if (this->config.wired_sync_mode == K4A_WIRED_SYNC_MODE_MASTER && !this->isSyncOutConnected()) + { + ofLogWarning(__FUNCTION__) << "Wired sync mode set to Master but Sync Out not connected! Reverting to Standalone."; + this->config.wired_sync_mode = K4A_WIRED_SYNC_MODE_STANDALONE; + } + else if (this->config.wired_sync_mode == K4A_WIRED_SYNC_MODE_SUBORDINATE && !this->isSyncInConnected()) + { + ofLogWarning(__FUNCTION__) << "Wired sync mode set to Subordinate but Sync In not connected! Reverting to Standalone."; + this->config.wired_sync_mode = K4A_WIRED_SYNC_MODE_STANDALONE; + } - if (this->config.wired_sync_mode != K4A_WIRED_SYNC_MODE_SUBORDINATE) - { - this->config.subordinate_delay_off_master_usec = 0; + if (this->config.wired_sync_mode != K4A_WIRED_SYNC_MODE_SUBORDINATE) + { + this->config.subordinate_delay_off_master_usec = 0; + } } // Start cameras. - try + if (bPlayback) { - this->device.start_cameras(&this->config); + playback->play(); } - catch (const k4a::error& e) + else { - ofLogError(__FUNCTION__) << e.what(); - return false; + try + { + this->device.start_cameras(&this->config); + + // Can only start the IMU if cameras are enabled + if (this->bEnableIMU) + { + k4a_device_start_imu(device.handle()); + } + } + catch (const k4a::error &e) + { + ofLogError(__FUNCTION__) << e.what(); + return false; + } + + } + + if (!bMultiDeviceSyncCapture && bEnableThread) { + this->startThread(); + } + else { + this->waitForThread(); } - this->startThread(); - ofAddListener(ofEvents().update, this, &Device::update); + if (this->config.color_format == K4A_IMAGE_FORMAT_COLOR_MJPG) { + this->decodeThread.start(); + } + else { + this->decodeThread.stop(); + } + + if (bEnableAutoUpdate) { + ofAddListener(ofEvents().update, this, &Device::update); + } this->bStreaming = true; @@ -265,17 +507,20 @@ namespace ofxAzureKinect bool Device::stopCameras() { - if (!this->bStreaming) return false; + if (!this->bStreaming) + return false; - std::unique_lock lock(this->mutex); - this->stopThread(); + std::unique_lock lock(this->timed_mtx); + this->decodeThread.stop(); + this->waitForThread(); this->condition.notify_all(); - ofRemoveListener(ofEvents().update, this, &Device::update); - - this->eventListeners.unsubscribeAll(); + if (bEnableAutoUpdate) { + ofRemoveListener(ofEvents().update, this, &Device::update); + } this->depthToWorldImg.reset(); + this->colorToWorldImg.reset(); this->transformation.destroy(); if (this->bUpdateBodies) @@ -285,13 +530,74 @@ namespace ofxAzureKinect this->bodyTracker = nullptr; } - this->device.stop_cameras(); + if (bPlayback) + { + playback->stop(); + playback->close(); + } + else + { + // Stop IMU if cameras are enabled + if (this->bEnableIMU) + { + k4a_device_stop_imu(device.handle()); + } + this->device.stop_cameras(); + } + + this->frameBack.reset(); + this->frameSwap.reset(); + this->frameFront.reset(); + this->depthToWorldPix.clear(); + this->colorToWorldPix.clear(); + this->depthTex.clear(); + this->colorTex.clear(); + this->irTex.clear(); + this->depthToWorldTex.clear(); + this->colorToWorldTex.clear(); + this->depthInColorTex.clear(); + this->colorInDepthTex.clear(); + this->bodyIndexTex.clear(); + this->pointCloudVbo.clear(); this->bStreaming = false; return true; } + void Device::update() + { + if (!bEnableThread && !bMultiDeviceSyncCapture) { + this->updatePixels(); + + if (this->lock()) { + this->frameBack.swapFrame(this->frameSwap); + this->bNewBuffer = true; + this->unlock(); + } + } + + this->bNewFrame = false; + + if (this->bNewBuffer) + { + if (this->decodeThread.isThreadRunning()) { + auto ret = this->decodeThread.update(this->frameSwap.colorPix, this->frameSwap.colorPixDeviceTime); + if (ret) { + this->frameSwap.bColorPixUpdated = true; + } + } + if (this->lock()) { + this->frameSwap.swapFrame(this->frameFront); + this->frameFront.bColorPixUpdated = false; + this->bNewBuffer = false; + this->unlock(); + } + this->updateTextures(); + this->condition.notify_all(); + } + } + bool Device::isSyncInConnected() const { return this->device.is_sync_in_connected(); @@ -306,66 +612,106 @@ namespace ofxAzureKinect { while (this->isThreadRunning()) { - std::unique_lock lock(this->mutex); - - while (this->isThreadRunning() && this->texFrameNum != this->pixFrameNum) - { - this->condition.wait(lock); + if (this->bAsyncJpegDecode) { + std::unique_lock lock(this->mutex); + if (this->isThreadRunning() && this->texFrameNum != this->pixFrameNum) + { + this->condition.wait_for(lock, std::chrono::milliseconds(100)); + } } this->updatePixels(); + + if (this->isThreadRunning() && this->tryLockFor(100)) { + this->frameBack.swapFrame(this->frameSwap); + this->bNewBuffer = true; + this->unlock(); + } + + std::this_thread::sleep_for(std::chrono::microseconds(100)); } + cerr << "done thread : " << getThreadName() << endl; } - void Device::update(ofEventArgs& args) + void Device::update(ofEventArgs &args) { - this->bNewFrame = false; - - if (this->texFrameNum != this->pixFrameNum) - { - std::unique_lock lock(this->mutex); - - this->updateTextures(); - - this->condition.notify_all(); - } + this->update(); } void Device::updatePixels() { // Get a capture. - try + if (this->bPlayback) { - if (!this->device.get_capture(&this->capture, std::chrono::milliseconds(TIMEOUT_IN_MS))) + if (playback->isPlaying()) + { + capture = k4a::capture(playback->getNextCapture()); + if (bEnableIMU) + { + imu_sample = playback->getNextImuSample(); + // printf(" | Accelerometer temperature:%.2f x:%.4f y:%.4f z: %.4f\n", + // imu_sample.temperature, + // imu_sample.acc_sample.xyz.x, + // imu_sample.acc_sample.xyz.y, + // imu_sample.acc_sample.xyz.z); + } + } + else if (playback->isPaused()) + { + playback->seek(); + capture = k4a::capture(playback->getNextCapture()); + } + else { - ofLogWarning(__FUNCTION__) << "Timed out waiting for a capture for device " << this->index << "::" << this->serialNumber << "."; + // if we are stopped, just return return; } } - catch (const k4a::error& e) + else if (this->bMultiDeviceSyncCapture && master_device_capture != nullptr) { + } + else { - ofLogError(__FUNCTION__) << e.what(); - return; + try + { + if (this->device.get_capture(&this->capture, std::chrono::milliseconds(TIMEOUT_IN_MS))) + { + if (bEnableIMU) { + this->device.get_imu_sample(&this->imu_sample); + } + } + else + { + ofLogWarning(__FUNCTION__) << "Timed out waiting for a capture for device " << this->index << "::" << this->serialNumber << "."; + return; + } + } + catch (const k4a::error &e) + { + ofLogError(__FUNCTION__) << e.what(); + return; + } } // Probe for a depth16 image. + auto& f = this->frameBack; auto depthImg = this->capture.get_depth_image(); if (depthImg) { const auto depthDims = glm::ivec2(depthImg.get_width_pixels(), depthImg.get_height_pixels()); - if (!depthPix.isAllocated()) + if (!f.depthPix.isAllocated()) { - this->depthPix.allocate(depthDims.x, depthDims.y, 1); + f.depthPix.allocate(depthDims.x, depthDims.y, 1); } - const auto depthData = reinterpret_cast(depthImg.get_buffer()); - this->depthPix.setFromPixels(depthData, depthDims.x, depthDims.y, 1); + const auto depthData = reinterpret_cast(depthImg.get_buffer()); + f.depthPix.setFromPixels(depthData, depthDims.x, depthDims.y, 1); + f.depthPixDeviceTime = depthImg.get_device_timestamp(); ofLogVerbose(__FUNCTION__) << "Capture Depth16 " << depthDims.x << "x" << depthDims.y << " stride: " << depthImg.get_stride_bytes() << "."; } else { - ofLogWarning(__FUNCTION__) << "No Depth16 capture found (" << ofGetFrameNum() << ")!"; + ofLogWarning(__FUNCTION__) << "No Depth16 capture found!"; } k4a::image colorImg; @@ -376,34 +722,49 @@ namespace ofxAzureKinect if (colorImg) { const auto colorDims = glm::ivec2(colorImg.get_width_pixels(), colorImg.get_height_pixels()); - if (!colorPix.isAllocated()) + if (!f.colorPix.isAllocated()) { - this->colorPix.allocate(colorDims.x, colorDims.y, OF_PIXELS_BGRA); + f.colorPix.allocate(colorDims.x, colorDims.y, OF_PIXELS_BGRA); } if (this->config.color_format == K4A_IMAGE_FORMAT_COLOR_MJPG) { - const int decompressStatus = tjDecompress2(this->jpegDecompressor, - colorImg.get_buffer(), - static_cast(colorImg.get_size()), - this->colorPix.getData(), - colorDims.x, - 0, // pitch - colorDims.y, - TJPF_BGRA, - TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE); + if (this->bAsyncJpegDecode) { + JpegTask task; + task.colorPixBuf.set((const char*)colorImg.get_buffer(), colorImg.get_size()); + task.colorPixDeviceTime = colorImg.get_device_timestamp(); + this->decodeThread.pushTaskIfEmpty(task); + f.bColorPixUpdated = false; + } + else { + const int decompressStatus = tjDecompress2(this->jpegDecompressor, + colorImg.get_buffer(), + static_cast(colorImg.get_size()), + f.colorPix.getData(), + colorDims.x, + 0, // pitch + colorDims.y, + TJPF_BGRA, + TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE); + f.bColorPixUpdated = true; + } } else { - const auto colorData = reinterpret_cast(colorImg.get_buffer()); - this->colorPix.setFromPixels(colorData, colorDims.x, colorDims.y, 4); + const auto colorData = reinterpret_cast(colorImg.get_buffer()); + f.colorPix.setFromPixels(colorData, colorDims.x, colorDims.y, 4); + f.bColorPixUpdated = true; + } + + if (f.bColorPixUpdated) { + f.colorPixDeviceTime = colorImg.get_device_timestamp(); } ofLogVerbose(__FUNCTION__) << "Capture Color " << colorDims.x << "x" << colorDims.y << " stride: " << colorImg.get_stride_bytes() << "."; } else { - ofLogWarning(__FUNCTION__) << "No Color capture found (" << ofGetFrameNum() << ")!"; + ofLogWarning(__FUNCTION__) << "No Color capture found!"; } } @@ -415,86 +776,63 @@ namespace ofxAzureKinect if (irImg) { const auto irSize = glm::ivec2(irImg.get_width_pixels(), irImg.get_height_pixels()); - if (!this->irPix.isAllocated()) + if (!f.irPix.isAllocated()) { - this->irPix.allocate(irSize.x, irSize.y, 1); + f.irPix.allocate(irSize.x, irSize.y, 1); } - const auto irData = reinterpret_cast(irImg.get_buffer()); - this->irPix.setFromPixels(irData, irSize.x, irSize.y, 1); + const auto irData = reinterpret_cast(irImg.get_buffer()); + f.irPix.setFromPixels(irData, irSize.x, irSize.y, 1); ofLogVerbose(__FUNCTION__) << "Capture Ir16 " << irSize.x << "x" << irSize.y << " stride: " << irImg.get_stride_bytes() << "."; } else { - ofLogWarning(__FUNCTION__) << "No Ir16 capture found (" << ofGetFrameNum() << ")!"; + ofLogWarning(__FUNCTION__) << "No Ir16 capture found!"; } } if (this->bUpdateBodies) { - k4a_wait_result_t enqueueResult = k4abt_tracker_enqueue_capture(this->bodyTracker, this->capture.handle(), K4A_WAIT_INFINITE); - if (enqueueResult == K4A_WAIT_RESULT_FAILED) + tracker.update(capture.handle()); + } + + if (this->bUpdateVbo) + { + if (this->bUpdateColor) { - ofLogError(__FUNCTION__) << "Failed adding capture to tracker process queue!"; + this->updatePointsCache(depthImg, depthToWorldImg); //(colorImg, this->colorToWorldImg); } else { - k4abt_frame_t bodyFrame = nullptr; - k4a_wait_result_t popResult = k4abt_tracker_pop_result(this->bodyTracker, &bodyFrame, K4A_WAIT_INFINITE); - if (popResult == K4A_WAIT_RESULT_SUCCEEDED) - { - // Probe for a body index map image. - k4a::image bodyIndexImg = k4abt_frame_get_body_index_map(bodyFrame); - const auto bodyIndexSize = glm::ivec2(bodyIndexImg.get_width_pixels(), bodyIndexImg.get_height_pixels()); - if (!this->bodyIndexPix.isAllocated()) - { - this->bodyIndexPix.allocate(bodyIndexSize.x, bodyIndexSize.y, 1); - } - - const auto bodyIndexData = reinterpret_cast(bodyIndexImg.get_buffer()); - this->bodyIndexPix.setFromPixels(bodyIndexData, bodyIndexSize.x, bodyIndexSize.y, 1); - - ofLogVerbose(__FUNCTION__) << "Capture BodyIndex " << bodyIndexSize.x << "x" << bodyIndexSize.y << " stride: " << bodyIndexImg.get_stride_bytes() << "."; - bodyIndexImg.reset(); - - size_t numBodies = k4abt_frame_get_num_bodies(bodyFrame); - ofLogVerbose(__FUNCTION__) << numBodies << " bodies found!"; - - this->bodySkeletons.resize(numBodies); - this->bodyIDs.resize(numBodies); - for (size_t i = 0; i < numBodies; i++) - { - k4abt_skeleton_t skeleton; - k4abt_frame_get_body_skeleton(bodyFrame, i, &skeleton); - this->bodySkeletons[i] = skeleton; - uint32_t id = k4abt_frame_get_body_id(bodyFrame, i); - this->bodyIDs[i] = id; - } - - // Release body frame once we're finished. - k4abt_frame_release(bodyFrame); - } + this->updatePointsCache(depthImg, depthToWorldImg); } } - if (this->bUpdateVbo) + if (colorImg && this->bUpdateColor && (this->config.color_format == K4A_IMAGE_FORMAT_COLOR_BGRA32 || (!bAsyncJpegDecode))) { - if (this->bUpdateColor) + if (this->config.color_format == K4A_IMAGE_FORMAT_COLOR_BGRA32) { - this->updatePointsCache(colorImg, this->colorToWorldImg); + // TODO: Fix this for non-BGRA formats, maybe always keep a BGRA k4a::image around. + this->updateDepthInColorFrame(depthImg, colorImg); + this->updateColorInDepthFrame(depthImg, colorImg); } else { - this->updatePointsCache(depthImg, this->depthToWorldImg); + k4a::image bgraColorImg = k4a::image::create_from_buffer( + K4A_IMAGE_FORMAT_COLOR_BGRA32, f.colorPix.getWidth(), f.colorPix.getHeight(), + f.colorPix.getBytesStride(), f.colorPix.getData(), f.colorPix.size(), nullptr, nullptr); + // TODO: Fix this for non-BGRA formats, maybe always keep a BGRA k4a::image around. + this->updateDepthInColorFrame(depthImg, bgraColorImg); + this->updateColorInDepthFrame(depthImg, bgraColorImg); } } - if (colorImg && this->bUpdateColor && this->config.color_format == K4A_IMAGE_FORMAT_COLOR_BGRA32) + // Do any recording before releasing the capture + if (this->bRecording) { - // TODO: Fix this for non-BGRA formats, maybe always keep a BGRA k4a::image around. - this->updateDepthInColorFrame(depthImg, colorImg); - this->updateColorInDepthFrame(depthImg, colorImg); + k4a_capture_t capture_handle = capture.handle(); + recording->record(&capture_handle); } // Release images. @@ -511,25 +849,26 @@ namespace ofxAzureKinect void Device::updateTextures() { - if (this->depthPix.isAllocated()) + auto& f = this->frameFront; + if (f.depthPix.isAllocated()) { // Update the depth texture. if (!this->depthTex.isAllocated()) { - this->depthTex.allocate(this->depthPix); + this->depthTex.allocate(f.depthPix); this->depthTex.setTextureMinMagFilter(GL_NEAREST, GL_NEAREST); } - this->depthTex.loadData(this->depthPix); + this->depthTex.loadData(f.depthPix); ofLogVerbose(__FUNCTION__) << "Update Depth16 " << this->depthTex.getWidth() << "x" << this->depthTex.getHeight() << "."; } - if (this->bUpdateColor && this->colorPix.isAllocated()) + if (this->bUpdateColor && f.colorPix.isAllocated()) { // Update the color texture. if (!this->colorTex.isAllocated()) { - this->colorTex.allocate(this->colorPix); + this->colorTex.allocate(f.colorPix); this->colorTex.setTextureMinMagFilter(GL_NEAREST, GL_NEAREST); if (this->config.color_format == K4A_IMAGE_FORMAT_COLOR_BGRA32) @@ -543,59 +882,59 @@ namespace ofxAzureKinect } } - this->colorTex.loadData(this->colorPix); + this->colorTex.loadData(f.colorPix); ofLogVerbose(__FUNCTION__) << "Update Color " << this->colorTex.getWidth() << "x" << this->colorTex.getHeight() << "."; } - if (this->bUpdateIr && this->irPix.isAllocated()) + if (this->bUpdateIr && f.irPix.isAllocated()) { // Update the IR16 image. if (!this->irTex.isAllocated()) { - this->irTex.allocate(this->irPix); + this->irTex.allocate(f.irPix); this->irTex.setTextureMinMagFilter(GL_NEAREST, GL_NEAREST); this->irTex.setRGToRGBASwizzles(true); } - this->irTex.loadData(this->irPix); + this->irTex.loadData(f.irPix); ofLogVerbose(__FUNCTION__) << "Update Ir16 " << this->irTex.getWidth() << "x" << this->irTex.getHeight() << "."; } - if (this->bUpdateBodies && this->bodyIndexPix.isAllocated()) + if (this->bUpdateBodies && f.bodyIndexPix.isAllocated()) { if (!this->bodyIndexTex.isAllocated()) { - this->bodyIndexTex.allocate(this->bodyIndexPix); + this->bodyIndexTex.allocate(f.bodyIndexPix); this->bodyIndexTex.setTextureMinMagFilter(GL_NEAREST, GL_NEAREST); } - this->bodyIndexTex.loadData(this->bodyIndexPix); + this->bodyIndexTex.loadData(f.bodyIndexPix); } if (this->bUpdateVbo) { - this->pointCloudVbo.setVertexData(this->positionCache.data(), this->numPoints, GL_STREAM_DRAW); - this->pointCloudVbo.setTexCoordData(this->uvCache.data(), this->numPoints, GL_STREAM_DRAW); + this->pointCloudVbo.setVertexData(f.positionCache.data(), f.numPoints, GL_STREAM_DRAW); + this->pointCloudVbo.setTexCoordData(f.uvCache.data(), f.numPoints, GL_STREAM_DRAW); } - if (this->bUpdateColor && this->config.color_format == K4A_IMAGE_FORMAT_COLOR_BGRA32) + if (this->bUpdateColor && (this->config.color_format == K4A_IMAGE_FORMAT_COLOR_BGRA32 || (!bAsyncJpegDecode))) { - if (this->depthInColorPix.isAllocated()) + if (f.depthInColorPix.isAllocated()) { if (!this->depthInColorTex.isAllocated()) { - this->depthInColorTex.allocate(this->depthInColorPix); + this->depthInColorTex.allocate(f.depthInColorPix); this->depthInColorTex.setTextureMinMagFilter(GL_NEAREST, GL_NEAREST); } - this->depthInColorTex.loadData(this->depthInColorPix); + this->depthInColorTex.loadData(f.depthInColorPix); } - if (this->colorInDepthPix.isAllocated()) + if (f.colorInDepthPix.isAllocated()) { if (!this->colorInDepthTex.isAllocated()) { - this->colorInDepthTex.allocate(this->colorInDepthPix); + this->colorInDepthTex.allocate(f.colorInDepthPix); this->colorInDepthTex.setTextureMinMagFilter(GL_NEAREST, GL_NEAREST); this->colorInDepthTex.bind(); { @@ -605,7 +944,7 @@ namespace ofxAzureKinect this->colorInDepthTex.unbind(); } - this->colorInDepthTex.loadData(this->colorInDepthPix); + this->colorInDepthTex.loadData(f.colorInDepthPix); } } @@ -664,9 +1003,9 @@ namespace ofxAzureKinect return false; } - bool Device::setupImageToWorldTable(k4a_calibration_type_t type, k4a::image& img) + bool Device::setupImageToWorldTable(k4a_calibration_type_t type, k4a::image &img) { - const k4a_calibration_camera_t& calibrationCamera = (type == K4A_CALIBRATION_TYPE_DEPTH) ? this->calibration.depth_camera_calibration : this->calibration.color_camera_calibration; + const k4a_calibration_camera_t &calibrationCamera = (type == K4A_CALIBRATION_TYPE_DEPTH) ? this->calibration.depth_camera_calibration : this->calibration.color_camera_calibration; const auto dims = glm::ivec2( calibrationCamera.resolution_width, @@ -675,16 +1014,16 @@ namespace ofxAzureKinect try { img = k4a::image::create(K4A_IMAGE_FORMAT_CUSTOM, - dims.x, dims.y, - dims.x * static_cast(sizeof(k4a_float2_t))); + dims.x, dims.y, + dims.x * static_cast(sizeof(k4a_float2_t))); } - catch (const k4a::error& e) + catch (const k4a::error &e) { ofLogError(__FUNCTION__) << e.what(); return false; } - auto imgData = reinterpret_cast(img.get_buffer()); + auto imgData = reinterpret_cast(img.get_buffer()); k4a_float2_t p; k4a_float3_t ray; @@ -717,7 +1056,8 @@ namespace ofxAzureKinect return true; } - bool Device::updatePointsCache(k4a::image& frameImg, k4a::image& tableImg) + // Kinect Thread function. + bool Device::updatePointsCache(k4a::image &frameImg, k4a::image &tableImg) { const auto frameDims = glm::ivec2(frameImg.get_width_pixels(), frameImg.get_height_pixels()); const auto tableDims = glm::ivec2(tableImg.get_width_pixels(), tableImg.get_height_pixels()); @@ -727,11 +1067,12 @@ namespace ofxAzureKinect return false; } - const auto frameData = reinterpret_cast(frameImg.get_buffer()); - const auto tableData = reinterpret_cast(tableImg.get_buffer()); + const auto frameData = reinterpret_cast(frameImg.get_buffer()); + const auto tableData = reinterpret_cast(tableImg.get_buffer()); - this->positionCache.resize(frameDims.x * frameDims.y); - this->uvCache.resize(frameDims.x * frameDims.y); + auto& f = this->frameBack; + f.positionCache.resize(frameDims.x * frameDims.y); + f.uvCache.resize(frameDims.x * frameDims.y); int count = 0; for (int y = 0; y < frameDims.y; ++y) @@ -743,25 +1084,25 @@ namespace ofxAzureKinect tableData[idx].xy.x != 0 && tableData[idx].xy.y != 0) { float depthVal = static_cast(frameData[idx]); - this->positionCache[count] = glm::vec3( + f.positionCache[count] = glm::vec3( tableData[idx].xy.x * depthVal, tableData[idx].xy.y * depthVal, - depthVal - ); + depthVal); - this->uvCache[count] = glm::vec2(x, y); + f.uvCache[count] = glm::vec2(x, y); ++count; } } } - this->numPoints = count; + f.numPoints = count; return true; } - bool Device::updateDepthInColorFrame(const k4a::image& depthImg, const k4a::image& colorImg) + // Kinect Thread function. + bool Device::updateDepthInColorFrame(const k4a::image &depthImg, const k4a::image &colorImg) { const auto colorDims = glm::ivec2(colorImg.get_width_pixels(), colorImg.get_height_pixels()); @@ -769,25 +1110,26 @@ namespace ofxAzureKinect try { transformedDepthImg = k4a::image::create(K4A_IMAGE_FORMAT_DEPTH16, - colorDims.x, colorDims.y, - colorDims.x * static_cast(sizeof(uint16_t))); + colorDims.x, colorDims.y, + colorDims.x * static_cast(sizeof(uint16_t))); this->transformation.depth_image_to_color_camera(depthImg, &transformedDepthImg); } - catch (const k4a::error& e) + catch (const k4a::error &e) { ofLogError(__FUNCTION__) << e.what(); return false; } - const auto transformedColorData = reinterpret_cast(transformedDepthImg.get_buffer()); + const auto transformedColorData = reinterpret_cast(transformedDepthImg.get_buffer()); - if (!this->depthInColorPix.isAllocated()) + auto& f = this->frameBack; + if (!f.depthInColorPix.isAllocated()) { - this->depthInColorPix.allocate(colorDims.x, colorDims.y, 1); + f.depthInColorPix.allocate(colorDims.x, colorDims.y, 1); } - this->depthInColorPix.setFromPixels(transformedColorData, colorDims.x, colorDims.y, 1); + f.depthInColorPix.setFromPixels(transformedColorData, colorDims.x, colorDims.y, 1); ofLogVerbose(__FUNCTION__) << "Depth in Color " << colorDims.x << "x" << colorDims.y << " stride: " << transformedDepthImg.get_stride_bytes() << "."; @@ -796,7 +1138,8 @@ namespace ofxAzureKinect return true; } - bool Device::updateColorInDepthFrame(const k4a::image& depthImg, const k4a::image& colorImg) + // Kinect Thread function. + bool Device::updateColorInDepthFrame(const k4a::image &depthImg, const k4a::image &colorImg) { const auto depthDims = glm::ivec2(depthImg.get_width_pixels(), depthImg.get_height_pixels()); @@ -804,26 +1147,27 @@ namespace ofxAzureKinect try { transformedColorImg = k4a::image::create(K4A_IMAGE_FORMAT_COLOR_BGRA32, - depthDims.x, depthDims.y, - depthDims.x * 4 * static_cast(sizeof(uint8_t))); + depthDims.x, depthDims.y, + depthDims.x * 4 * static_cast(sizeof(uint8_t))); this->transformation.color_image_to_depth_camera(depthImg, colorImg, &transformedColorImg); } - catch (const k4a::error& e) + catch (const k4a::error &e) { ofLogError(__FUNCTION__) << e.what(); return false; } - const auto transformedColorData = reinterpret_cast(transformedColorImg.get_buffer()); + const auto transformedColorData = reinterpret_cast(transformedColorImg.get_buffer()); - if (!this->colorInDepthPix.isAllocated()) + auto& f = this->frameBack; + if (!f.colorInDepthPix.isAllocated()) { - this->colorInDepthPix.allocate(depthDims.x, depthDims.y, OF_PIXELS_BGRA); + f.colorInDepthPix.allocate(depthDims.x, depthDims.y, OF_PIXELS_BGRA); } - this->colorInDepthPix.setFromPixels(transformedColorData, depthDims.x, depthDims.y, 4); - + f.colorInDepthPix.setFromPixels(transformedColorData, depthDims.x, depthDims.y, 4); + ofLogVerbose(__FUNCTION__) << "Color in Depth " << depthDims.x << "x" << depthDims.y << " stride: " << transformedColorImg.get_stride_bytes() << "."; transformedColorImg.reset(); @@ -846,108 +1190,385 @@ namespace ofxAzureKinect return this->bNewFrame; } - const std::string& Device::getSerialNumber() const + const std::string &Device::getSerialNumber() const { return this->serialNumber; } - const ofShortPixels& Device::getDepthPix() const + const ofShortPixels &Device::getDepthPix() const { - return this->depthPix; + return this->frameFront.depthPix; } - const ofTexture& Device::getDepthTex() const + const ofTexture &Device::getDepthTex() const { return this->depthTex; } - const ofPixels& Device::getColorPix() const + const ofPixels &Device::getColorPix() const { - return this->colorPix; + return this->frameFront.colorPix; } - const ofTexture& Device::getColorTex() const + const ofTexture &Device::getColorTex() const { return this->colorTex; } - const ofShortPixels& Device::getIrPix() const + const ofShortPixels &Device::getIrPix() const { - return this->irPix; + return this->frameFront.irPix; } - const ofTexture& Device::getIrTex() const + const ofTexture &Device::getIrTex() const { return this->irTex; } - const ofFloatPixels& Device::getDepthToWorldPix() const + const ofFloatPixels &Device::getDepthToWorldPix() const { return this->depthToWorldPix; } - const ofTexture& Device::getDepthToWorldTex() const + const ofTexture &Device::getDepthToWorldTex() const { return this->depthToWorldTex; } - const ofFloatPixels& Device::getColorToWorldPix() const + const ofFloatPixels &Device::getColorToWorldPix() const { return this->colorToWorldPix; } - const ofTexture& Device::getColorToWorldTex() const + const ofTexture &Device::getColorToWorldTex() const { return this->colorToWorldTex; } - const ofShortPixels& Device::getDepthInColorPix() const + const ofShortPixels &Device::getDepthInColorPix() const { - return this->depthInColorPix; + return this->frameFront.depthInColorPix; } - const ofTexture& Device::getDepthInColorTex() const + const ofTexture &Device::getDepthInColorTex() const { return this->depthInColorTex; } - const ofPixels& Device::getColorInDepthPix() const + const ofPixels &Device::getColorInDepthPix() const { - return this->colorInDepthPix; + return this->frameFront.colorInDepthPix; } - const ofTexture& Device::getColorInDepthTex() const + const ofTexture &Device::getColorInDepthTex() const { return this->colorInDepthTex; } - const ofPixels& Device::getBodyIndexPix() const + const ofVbo &Device::getPointCloudVbo() const { - return this->bodyIndexPix; + return this->pointCloudVbo; } - const ofTexture& Device::getBodyIndexTex() const + int32_t Device::getColorCameraControlValue(k4a_color_control_command_t command) const { - return this->bodyIndexTex; + k4a_color_control_mode_t mode; + int32_t ret; + this->device.get_color_control(command, &mode, &ret); + if (mode == K4A_COLOR_CONTROL_MODE_AUTO) { + return INT_MIN; + } + else { + return ret; + } } - size_t Device::getNumBodies() const + void Device::setColorCameraControlValue(k4a_color_control_command_t command, int32_t value) { - return this->bodySkeletons.size(); + this->device.set_color_control(command, K4A_COLOR_CONTROL_MODE_MANUAL, value); } - const std::vector& Device::getBodySkeletons() const + int32_t Device::getExposureTimeAbsolute() const { - return this->bodySkeletons; + return getColorCameraControlValue(K4A_COLOR_CONTROL_EXPOSURE_TIME_ABSOLUTE); } - const std::vector& Device::getBodyIDs() const + void Device::setExposureTimeAbsolute(int32_t exposure_usec) { - return this->bodyIDs; + setColorCameraControlValue(K4A_COLOR_CONTROL_EXPOSURE_TIME_ABSOLUTE, exposure_usec); } - const ofVbo& Device::getPointCloudVbo() const + void Device::startRecording(std::string filename, float delay) { - return this->pointCloudVbo; + if (isRecording()) { + return; + } + + recording = new Record(); + recording->setup(device.handle(), this->config, bEnableIMU, delay, filename); + recording->start(); + + bRecording = true; + } + + void Device::stopRecording() + { + if (recording) { + recording->stop(); + delete recording; + recording = nullptr; + } + bRecording = false; + } + + bool Device::isRecording() const + { + return bRecording; + } + + bool Device::lock() + { + timed_mtx.lock(); + return true; + } + + bool Device::tryLock() + { + return timed_mtx.try_lock(); + } + + bool Device::tryLockFor(uint64_t millisec) + { + return timed_mtx.try_lock_for(std::chrono::milliseconds(millisec)); + } + + void Device::unlock() + { + timed_mtx.unlock(); + } + + float Device::getRecordingTimerDelay() + { + if (recording != nullptr) + return recording->getTimerDelay(); + else + return -1; + } + + void Device::listener_playback_play(bool val) + { + playback->play(); + } + void Device::listener_playback_pause(bool val) + { + playback->pause(); + } + void Device::listener_playback_stop(bool val) + { + playback->stop(); + } + void Device::listener_playback_seek(float val) + { + playback->seek(val); + } + + + void MultiDeviceSyncCapture::setMasterDevice(Device * p) + { + master_device = p; + master_device->bMultiDeviceSyncCapture = true; + master_device->master_device_capture = this; + } + void MultiDeviceSyncCapture::addSubordinateDevice(Device * p) + { + p->bMultiDeviceSyncCapture = true; + p->master_device_capture = this; + subordinate_devices.push_back(p); + } + + void MultiDeviceSyncCapture::removeAllDevices() + { + if (master_device != nullptr) { + master_device->bMultiDeviceSyncCapture = false; + master_device->master_device_capture = nullptr; + master_device = nullptr; + } + + for (auto &p : subordinate_devices) + { + p->bMultiDeviceSyncCapture = false; + p->master_device_capture = nullptr; + } + subordinate_devices.clear(); + } + + void MultiDeviceSyncCapture::start() + { + if (master_device == nullptr || subordinate_devices.empty()) { + cerr << "MultiDeviceSyncCapture::setMasterDevice, addSubordinateDevice must be called before start." << endl; + return; + } + startThread(); + } + + void MultiDeviceSyncCapture::stop() + { + waitForThread(); + } + + void MultiDeviceSyncCapture::setMaxAllowableTimeOffsetUsec(uint32_t usec) + { + max_allowable_time_offset_error_for_image_timestamp = std::chrono::microseconds(usec); + } + + void MultiDeviceSyncCapture::threadedFunction() + { + while (isThreadRunning()) { + // Dealing with the synchronized cameras is complex. The Azure Kinect DK: + // (a) does not guarantee exactly equal timestamps between depth and color or between cameras (delays can + // be configured but timestamps will only be approximately the same) + // (b) does not guarantee that, if the two most recent images were synchronized, that calling get_capture + // just once on each camera will still be synchronized. + // There are several reasons for all of this. Internally, devices keep a queue of a few of the captured images + // and serve those images as requested by get_capture(). However, images can also be dropped at any moment, and + // one device may have more images ready than another device at a given moment, et cetera. + // + // Also, the process of synchronizing is complex. The cameras are not guaranteed to exactly match in all of + // their timestamps when synchronized (though they should be very close). All delays are relative to the master + // camera's color camera. To deal with these complexities, we employ a fairly straightforward algorithm. Start + // by reading in two captures, then if the camera images were not taken at roughly the same time read a new one + // from the device that had the older capture until the timestamps roughly match. + + // The captures used in the loop are outside of it so that they can persist across loop iterations. This is + // necessary because each time this loop runs we'll only update the older capture. + // The captures are stored in a vector where the first element of the vector is the master capture and + // subsequent elements are subordinate captures + //std::vector captures(subordinate_devices.size() + 1); // add 1 for the master + auto devices = subordinate_devices; + devices.push_back(master_device); + + size_t current_index = 0; + master_device->device.get_capture(&master_device->capture, std::chrono::milliseconds{ K4A_WAIT_INFINITE }); + if (master_device->bEnableIMU) { + master_device->device.get_imu_sample(&master_device->imu_sample); + } + ++current_index; + for (auto &d : subordinate_devices) + { + d->device.get_capture(&d->capture, std::chrono::milliseconds{ K4A_WAIT_INFINITE }); + if (d->bEnableIMU) { + d->device.get_imu_sample(&d->imu_sample); + } + ++current_index; + } + + bool have_synced_images = false; + std::chrono::system_clock::time_point start = std::chrono::system_clock::now(); + while (!have_synced_images) + { + // Timeout if this is taking too long + int64_t duration_ms = + std::chrono::duration_cast(std::chrono::system_clock::now() - start).count(); + if (duration_ms > WAIT_FOR_SYNCHRONIZED_CAPTURE_TIMEOUT) + { + cerr << "ERROR: Timedout waiting for synchronized captures\n"; + } + + k4a::image master_color_image = master_device->capture.get_color_image(); + std::chrono::microseconds master_color_image_time = master_color_image.get_device_timestamp(); + + for (size_t i = 0; i < subordinate_devices.size(); ++i) + { + k4a::image sub_image; + if (compare_sub_depth_instead_of_color) + { + sub_image = subordinate_devices[i]->capture.get_depth_image(); + } + else + { + sub_image = subordinate_devices[i]->capture.get_color_image(); + } + + if (master_color_image && sub_image) + { + std::chrono::microseconds sub_image_time = sub_image.get_device_timestamp(); + // The subordinate's color image timestamp, ideally, is the master's color image timestamp plus the + // delay we configured between the master device color camera and subordinate device color camera + std::chrono::microseconds expected_sub_image_time = + master_color_image_time + + std::chrono::microseconds{ subordinate_devices[i]->config.subordinate_delay_off_master_usec } + + std::chrono::microseconds{ subordinate_devices[i]->config.depth_delay_off_color_usec }; + std::chrono::microseconds sub_image_time_error = sub_image_time - expected_sub_image_time; + // The time error's absolute value must be within the permissible range. So, for example, if + // MAX_ALLOWABLE_TIME_OFFSET_ERROR_FOR_IMAGE_TIMESTAMP is 2, offsets of -2, -1, 0, 1, and -2 are + // permitted + if (sub_image_time_error < -max_allowable_time_offset_error_for_image_timestamp) + { + // Example, where MAX_ALLOWABLE_TIME_OFFSET_ERROR_FOR_IMAGE_TIMESTAMP is 1 + // time t=1 t=2 t=3 + // actual timestamp x . . + // expected timestamp . . x + // error: 1 - 3 = -2, which is less than the worst-case-allowable offset of -1 + // the subordinate camera image timestamp was earlier than it is allowed to be. This means the + // subordinate is lagging and we need to update the subordinate to get the subordinate caught up + log_lagging_time("sub", master_device->capture, subordinate_devices[i]->capture); + subordinate_devices[i]->device.get_capture(&subordinate_devices[i]->capture, + std::chrono::milliseconds{ K4A_WAIT_INFINITE }); + break; + } + else if (sub_image_time_error > max_allowable_time_offset_error_for_image_timestamp) + { + // Example, where MAX_ALLOWABLE_TIME_OFFSET_ERROR_FOR_IMAGE_TIMESTAMP is 1 + // time t=1 t=2 t=3 + // actual timestamp . . x + // expected timestamp x . . + // error: 3 - 1 = 2, which is more than the worst-case-allowable offset of 1 + // the subordinate camera image timestamp was later than it is allowed to be. This means the + // subordinate is ahead and we need to update the master to get the master caught up + log_lagging_time("master", master_device->capture, subordinate_devices[i]->capture); + master_device->device.get_capture(&master_device->capture, std::chrono::milliseconds{ K4A_WAIT_INFINITE }); + break; + } + else + { + // These captures are sufficiently synchronized. If we've gotten to the end, then all are + // synchronized. + if (i == subordinate_devices.size() - 1) + { + log_synced_image_time(master_device->capture, subordinate_devices[i]->capture); + have_synced_images = true; // now we'll finish the for loop and then exit the while loop + } + } + } + else if (!master_color_image) + { + std::cout << "Master image was bad!\n"; + master_device->device.get_capture(&master_device->capture, std::chrono::milliseconds{ K4A_WAIT_INFINITE }); + break; + } + else if (!sub_image) + { + std::cout << "Subordinate image was bad!" << endl; + subordinate_devices[i]->device.get_capture(&subordinate_devices[i]->capture, + std::chrono::milliseconds{ K4A_WAIT_INFINITE }); + break; + } + } + } + // if we've made it to here, it means that we have synchronized captures. + for (auto& device : devices) { + device->updatePixels(); + } + for (auto& device : devices) { + if (device->tryLockFor(100)) { + device->frameBack.swapFrame(device->frameSwap); + device->bNewBuffer = true; + device->unlock(); + } + } + + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + std::cout << "Multi Device Sync Capture thread finished" << endl; } -} +} // namespace ofxAzureKinect diff --git a/src/ofxAzureKinect/Device.h b/src/ofxAzureKinect/Device.h index a79714b..0825a35 100644 --- a/src/ofxAzureKinect/Device.h +++ b/src/ofxAzureKinect/Device.h @@ -15,6 +15,9 @@ #include "ofVectorMath.h" #include "Types.h" +#include "Record.h" +#include "Playback.h" +#include "BodyTracker.h" namespace ofxAzureKinect { @@ -36,6 +39,8 @@ namespace ofxAzureKinect bool syncImages; + bool enableIMU; + DeviceSettings(int idx = 0); }; @@ -50,10 +55,71 @@ namespace ofxAzureKinect BodyTrackingSettings(); }; - class Device - : ofThread + class MultiDeviceSyncCapture; + + class Device + : public ofThread { + protected: + struct Frame + { + ofShortPixels depthPix; + std::chrono::microseconds depthPixDeviceTime; + + ofPixels colorPix; + bool bColorPixUpdated = false; + std::chrono::microseconds colorPixDeviceTime; + + ofShortPixels irPix; + ofShortPixels depthInColorPix; + ofPixels colorInDepthPix; + + ofPixels bodyIndexPix; + std::vector bodySkeletons; + std::vector bodyIDs; + + std::vector positionCache; + std::vector uvCache; + size_t numPoints; + + void swapFrame(Frame& f); + void reset(); + }; + + struct JpegTask + { + ofBuffer colorPixBuf; + std::chrono::microseconds colorPixDeviceTime; + }; + + struct DecodedPix + { + ofPixels colorPix; + std::chrono::microseconds colorPixDeviceTime; + }; + + struct JpegDecodeThread : public ofThread + { + protected: + tjhandle jpegDecompressor; + ofThreadChannel toProcess; + ofThreadChannel processed; + void threadedFunction() override; + + public: + + JpegDecodeThread(); + ~JpegDecodeThread(); + + void start(); + void stop(); + + bool pushTaskIfEmpty(JpegTask& b); + bool update(ofPixels& outPix, std::chrono::microseconds& outTime); + } decodeThread; public: + friend class MultiDeviceSyncCapture; + static int getInstalledCount(); public: @@ -62,11 +128,15 @@ namespace ofxAzureKinect bool open(uint32_t idx = 0); bool open(const std::string& serialNumber); + + bool load(string filename); bool close(); bool startCameras(DeviceSettings deviceSettings = DeviceSettings(), BodyTrackingSettings bodyTrackingSettings = BodyTrackingSettings()); bool stopCameras(); + virtual void update(); + bool isSyncInConnected() const; bool isSyncOutConnected() const; @@ -74,64 +144,98 @@ namespace ofxAzureKinect bool isStreaming() const; bool isFrameNew() const; - const std::string& getSerialNumber() const; + const std::string &getSerialNumber() const; + + const ofShortPixels &getDepthPix() const; + const ofTexture &getDepthTex() const; + const std::chrono::microseconds& getDepthTexDeviceTime() const { + return this->frameFront.depthPixDeviceTime; + } - const ofShortPixels& getDepthPix() const; - const ofTexture& getDepthTex() const; + const ofPixels &getColorPix() const; + const ofTexture &getColorTex() const; + const std::chrono::microseconds& getColorTexDeviceTime() const { + return this->frameFront.colorPixDeviceTime; + } - const ofPixels& getColorPix() const; - const ofTexture& getColorTex() const; + const ofShortPixels &getIrPix() const; + const ofTexture &getIrTex() const; - const ofShortPixels& getIrPix() const; - const ofTexture& getIrTex() const; + const ofFloatPixels &getDepthToWorldPix() const; + const ofTexture &getDepthToWorldTex() const; - const ofFloatPixels& getDepthToWorldPix() const; - const ofTexture& getDepthToWorldTex() const; + const ofFloatPixels &getColorToWorldPix() const; + const ofTexture &getColorToWorldTex() const; - const ofFloatPixels& getColorToWorldPix() const; - const ofTexture& getColorToWorldTex() const; + const ofShortPixels &getDepthInColorPix() const; + const ofTexture &getDepthInColorTex() const; - const ofShortPixels& getDepthInColorPix() const; - const ofTexture& getDepthInColorTex() const; + const ofPixels &getColorInDepthPix() const; + const ofTexture &getColorInDepthTex() const; - const ofPixels& getColorInDepthPix() const; - const ofTexture& getColorInDepthTex() const; + const ofVbo &getPointCloudVbo() const; - const ofPixels& getBodyIndexPix() const; - const ofTexture& getBodyIndexTex() const; + BodyTracker *getBodyTracker() { return &tracker; } - size_t getNumBodies() const; - const std::vector& getBodySkeletons() const; - const std::vector& getBodyIDs() const; + int32_t getColorCameraControlValue(k4a_color_control_command_t command) const; + void setColorCameraControlValue(k4a_color_control_command_t command, int32_t value); - const ofVbo& getPointCloudVbo() const; + int32_t getExposureTimeAbsolute() const; + void setExposureTimeAbsolute(int32_t exposure_usec); + void startRecording(std::string filename = "", float delay = 0.0f); + void stopRecording(); + bool isRecording() const; + + bool isAsyncJpegDecode() const { return this->bAsyncJpegDecode; } + void setAsyncJpegDecode(bool b) { this->bAsyncJpegDecode = b; } + + bool isEnableAutoUpdate() const { return this->bEnableAutoUpdate; } + void setEnableAutoUpdate(bool b) { this->bEnableAutoUpdate = b; } + + bool isEnableThread() const { return this->bEnableThread; } + void setEnableThread(bool b) { this->bEnableThread = b; } + + k4a_imu_sample_t getIMUSample() const { return imu_sample; } + + bool lock(); + bool tryLock(); + bool tryLockFor(uint64_t millisec); + void unlock(); public: - ofParameter jointSmoothing{ "Joint Smoothing", 0.0f, 0.0f, 1.0f }; + float getRecordingTimerDelay(); + ofParameter play{"play", false}; + ofParameter pause{"pause", false}; + ofParameter stop{"stop", false}; + ofParameter seek{"Seek", 0.0f, 0.0f, 1.0f}; + + Playback* getPlayback() {return this->playback; } + Record* getRecord() { return this->recording; } protected: - void threadedFunction() override; + virtual void threadedFunction() override; - private: - void updatePixels(); - void updateTextures(); + virtual void updatePixels(); + virtual void updateTextures(); - void update(ofEventArgs& args); + virtual void update(ofEventArgs &args); bool setupDepthToWorldTable(); bool setupColorToWorldTable(); - bool setupImageToWorldTable(k4a_calibration_type_t type, k4a::image& img); + bool setupImageToWorldTable(k4a_calibration_type_t type, k4a::image &img); - bool updatePointsCache(k4a::image& frameImg, k4a::image& tableImg); + bool updatePointsCache(k4a::image &frameImg, k4a::image &tableImg); - bool updateDepthInColorFrame(const k4a::image& depthImg, const k4a::image& colorImg); - bool updateColorInDepthFrame(const k4a::image& depthImg, const k4a::image& colorImg); + bool updateDepthInColorFrame(const k4a::image &depthImg, const k4a::image &colorImg); + bool updateColorInDepthFrame(const k4a::image &depthImg, const k4a::image &colorImg); - private: + protected: int index; bool bOpen; bool bStreaming; - bool bNewFrame; + bool bPlayback; + bool bRecording; + bool bEnableIMU; bool bUpdateColor; bool bUpdateIr; @@ -139,10 +243,19 @@ namespace ofxAzureKinect bool bUpdateWorld; bool bUpdateVbo; - std::condition_variable condition; + bool bMultiDeviceSyncCapture; + bool bAsyncJpegDecode; + bool bEnableAutoUpdate; + bool bEnableThread; + + std::timed_mutex timed_mtx; + std::condition_variable_any condition; uint64_t pixFrameNum; uint64_t texFrameNum; + bool bNewBuffer; + bool bNewFrame; + std::string serialNumber; k4a_device_configuration_t config; @@ -156,39 +269,74 @@ namespace ofxAzureKinect tjhandle jpegDecompressor; - ofShortPixels depthPix; - ofTexture depthTex; - - ofPixels colorPix; - ofTexture colorTex; + k4a_imu_sample_t imu_sample; - ofShortPixels irPix; - ofTexture irTex; + // triple buffer + Frame frameBack; + Frame frameSwap; + Frame frameFront; + // these are thread safe k4a::image depthToWorldImg; ofFloatPixels depthToWorldPix; - ofTexture depthToWorldTex; - k4a::image colorToWorldImg; ofFloatPixels colorToWorldPix; - ofTexture colorToWorldTex; - ofShortPixels depthInColorPix; + ofTexture depthTex; + ofTexture colorTex; + ofTexture irTex; + ofTexture depthToWorldTex; + ofTexture colorToWorldTex; ofTexture depthInColorTex; - - ofPixels colorInDepthPix; ofTexture colorInDepthTex; - - ofPixels bodyIndexPix; ofTexture bodyIndexTex; - std::vector bodySkeletons; - std::vector bodyIDs; - - std::vector positionCache; - std::vector uvCache; - size_t numPoints; ofVbo pointCloudVbo; ofEventListeners eventListeners; + + BodyTracker tracker; + + Record *recording; + Playback *playback; + void listener_playback_play(bool val); + void listener_playback_pause(bool val); + void listener_playback_stop(bool val); + void listener_playback_seek(float val); + + MultiDeviceSyncCapture* master_device_capture = nullptr; + }; + + // reference implementation + // https://github.com/microsoft/Azure-Kinect-Sensor-SDK/blob/develop/examples/green_screen/MultiDeviceCapturer.h + // + // Note for use this class. + // device firmware version must be matched and latest. + // set manual exposure time to shorter, for acquiring correct sync. + // more info on : https://github.com/microsoft/Azure-Kinect-Sensor-SDK/issues/1261 + // + // Not sophisticated implementation. + class MultiDeviceSyncCapture : public ofThread + { + protected: + friend class Device; + Device* master_device = nullptr; + std::vector subordinate_devices; + bool compare_sub_depth_instead_of_color = false; + std::chrono::microseconds max_allowable_time_offset_error_for_image_timestamp = std::chrono::microseconds(1000); + public: + + // these funcs must be called before Device::startCameras(); + void setMasterDevice(Device* p); + void addSubordinateDevice(Device* p); + + void removeAllDevices(); + + void start(); + void stop(); + + void setMaxAllowableTimeOffsetUsec(uint32_t usec); + + protected: + void threadedFunction() override; }; -} \ No newline at end of file +} // namespace ofxAzureKinect \ No newline at end of file diff --git a/src/ofxAzureKinect/Playback.cpp b/src/ofxAzureKinect/Playback.cpp new file mode 100644 index 0000000..5c8d686 --- /dev/null +++ b/src/ofxAzureKinect/Playback.cpp @@ -0,0 +1,166 @@ +#include "Playback.h" + +namespace ofxAzureKinect +{ + + bool Playback::load(string _filename) + { + // Convert filename from string to char* + char *temp = new char[_filename.size() + 1]; + copy(_filename.begin(), _filename.end(), temp); + temp[_filename.size()] = '\0'; + + // Set the filename for recording output + this->filename = temp; + + if (k4a_playback_open(this->filename, &playback) != K4A_RESULT_SUCCEEDED) + { + ofLogError(__FUNCTION__) << "Failed to open recording: " << filename; + k4a_failed = true; + return false; + } + + recording_length = k4a_playback_get_recording_length_usec(playback); + printf("Recording is %lld seconds long\n", recording_length / 1000000); + + return true; + } + + void Playback::play() + { + status = PLAYING; + cout << "play" << endl; + } + + void Playback::pause() + { + status = PAUSED; + cout << "pause" << endl; + } + + void Playback::stop() + { + seek(0); + status = STOPPED; + cout << "stop" << endl; + } + + k4a_capture_t Playback::getNextCapture() + { + k4a_stream_result_t result = k4a_playback_get_next_capture(playback, &capture); + if (result == K4A_STREAM_RESULT_SUCCEEDED) + { + return capture; + } + else if (result == K4A_STREAM_RESULT_EOF) + { + // End of file reached + // ofLog() << "End of file reached." << endl; + seek(0); + return getNextCapture(); + } + else if (result == K4A_STREAM_RESULT_FAILED) + { + ofLogError(__FUNCTION__) << "Failed to read entire recording." << endl; + return nullptr; + } + return nullptr; + } + + k4a_imu_sample_t Playback::getNextImuSample() + { + // k4a_imu_sample_t imu_sample; + // checking for the IMU tracking isn't working ... may have to add a special tag when recording? + // cout << "IMU TRACK: " << k4a_playback_check_track_exists (playback, "K4A_IMU_TRACK") << endl; + if (config.imu_track_enabled) + k4a_playback_get_next_imu_sample(playback, &imu_sample); + return imu_sample; + } + + void Playback::seek(float amt) + { + b_seek_by_device_time = false; + seek_head = amt; + int64_t play_head = int64_t(ofMap(seek_head, 0, 1, 0, recording_length, true)); + + // Seek to 10 seconds from the start + if (k4a_playback_seek_timestamp(playback, play_head, K4A_PLAYBACK_SEEK_BEGIN) != K4A_RESULT_SUCCEEDED) + { + ofLogError(__FUNCTION__) << "K4A_PLAYBACK_SEEK FAILED."; + return; + } + } + + void Playback::seekByDeviceTime(int64_t device_usec) + { + b_seek_by_device_time = true; + last_seek_device_usec = device_usec; + + if (k4a_playback_seek_timestamp(playback, last_seek_device_usec, K4A_PLAYBACK_SEEK_DEVICE_TIME) != K4A_RESULT_SUCCEEDED) + { + ofLogError(__FUNCTION__) << "K4A_PLAYBACK_SEEK FAILED."; + return; + } + } + + void Playback::seek() + { + if (b_seek_by_device_time) { + this->seekByDeviceTime(last_seek_device_usec); + } + else + { + this->seek(this->seek_head); + } + } + + void Playback::close() + { + status = STOPPED; + k4a_playback_close(playback); + } + + k4a_record_configuration_t Playback::getDeviceSettings() + { + k4a_playback_get_record_configuration(playback, &config); + return config; + } + + k4a_calibration_t Playback::getCalibration() + { + k4a_calibration_t calibration; + if (K4A_RESULT_SUCCEEDED != k4a_playback_get_calibration(playback, &calibration)) + { + ofLogError(__FUNCTION__) << "Failed to get calibration data from recording."; + } + return calibration; + } + + string Playback::getSerialNumber() + { + return get_tag("K4A_DEVICE_SERIAL_NUMBER"); + } + + string Playback::get_tag(string tag_name) + { + char result_buffer[256]; + size_t result_size = 256; + + k4a_buffer_result_t result = k4a_playback_get_tag( + playback, tag_name.c_str(), result_buffer, &result_size); + if (K4A_BUFFER_RESULT_SUCCEEDED == result) + { + return result_buffer; + } + else if (K4A_BUFFER_RESULT_TOO_SMALL == result) + { + ofLogError(__FUNCTION__) << "Tag's {" << tag_name << "} has content that is too long."; + } + else + { + ofLogError(__FUNCTION__) << "Tag {" << tag_name << "} does not exist."; + } + return ""; + } + +} // namespace ofxAzureKinect \ No newline at end of file diff --git a/src/ofxAzureKinect/Playback.h b/src/ofxAzureKinect/Playback.h new file mode 100644 index 0000000..6a3ebc8 --- /dev/null +++ b/src/ofxAzureKinect/Playback.h @@ -0,0 +1,76 @@ +#pragma once + +#include + +#include +#include + +#include "ofMain.h" + + +namespace ofxAzureKinect +{ + class Playback + { + + public: + Playback(){}; + ~Playback(){}; + + bool load(string _filename); + k4a_record_configuration_t getDeviceSettings(); + string getSerialNumber(); + k4a_calibration_t getCalibration(); + + k4a_capture_t getNextCapture(); + k4a_imu_sample_t getNextImuSample(); + + void seek(); + void seek(float t); + void seekByDeviceTime(int64_t device_usec); + + void play(); + void pause(); + void stop(); + void close(); + + bool isPlaying() { return status == PLAYING; } + bool isPaused() { return status == PAUSED; } + bool isStopped() { return status == STOPPED; } + + int64_t getStartTimeOffsetUsec() const { + return config.start_timestamp_offset_usec; + } + uint64_t getRecordingLengthUsec() const { + return recording_length; + } + + private: + k4a_playback_t playback; + k4a_capture_t capture; + k4a_imu_sample_t imu_sample; + k4a_record_configuration_t config; + + char *filename; + bool k4a_failed = false; + + uint64_t recording_length = 0; + float seek_head = 0; + + float in = 0, out = 0; + + string get_tag(string tag_name); + string serial_number; + + enum Status + { + STOPPED, + PAUSED, + PLAYING + }; + Status status = STOPPED; + + bool b_seek_by_device_time = false; + int64_t last_seek_device_usec = 0; + }; +} // namespace ofxAzureKinect \ No newline at end of file diff --git a/src/ofxAzureKinect/Record.cpp b/src/ofxAzureKinect/Record.cpp new file mode 100644 index 0000000..0c5af01 --- /dev/null +++ b/src/ofxAzureKinect/Record.cpp @@ -0,0 +1,164 @@ +#include "Record.h" + +namespace ofxAzureKinect +{ + + void Record::setup(k4a_device_t device, k4a_device_configuration_t config, bool recording_imu_enabled, float delay, string filename) + { + this->device = device; + this->recording_imu_enabled = recording_imu_enabled; + this->delay = delay; + + // Set a timestamped or user defined filename + if (filename == "") + filename = ofToDataPath("output_" + ofGetTimestampString("%Y%m%d_%H-%M-%S") + ".mkv"); + else + { + filename = ofToDataPath(filename); + } + + // Convert filename from string to char* + char *temp = new char[filename.size() + 1]; + copy(filename.begin(), filename.end(), temp); + temp[filename.size()] = '\0'; + + // Set the filename for recording output + this->filename = temp; + + // Set up the default recording tracks + if (K4A_FAILED(k4a_record_create(this->filename, device, config, &recording))) + { + printf("Unable to create recording file: %s\n", this->filename); + k4a_failed = true; + return; + } + + // Add any custom tracks and tags you want to record + // See example code: https://github.com/microsoft/Azure-Kinect-Sensor-SDK/blob/develop/examples/k4arecord_custom_track/main.c + // ... + + // Add IMU track + if (this->recording_imu_enabled) + { + k4a_record_add_imu_track(recording); + } + + // Write the recording header after all the track metadata is set up. + k4a_record_write_header(recording); + + cout << "Recording Setup to file: " << this->filename << endl; + } + + void Record::start() + { + delay_start = ofGetElapsedTimef(); + record_frame_count = 0; + if (delay != 0) + { + cout << "Recording Will Begin In " << delay << " seconds!" << endl; + } + } + + void Record::stop() + { + if (k4a_failed) + { + ofLogError(__FUNCTION__) << "Recording Failed ... see error above."; + } + else + { + cout << "\nSaving Recording to: " << filename << endl; + k4a_record_flush(recording); + k4a_record_close(recording); + cout << "Done." << endl; + } + } + + void Record::record(k4a_capture_t *capture) + { + if (k4a_failed) + { + ofLogError(__FUNCTION__) << "Recording Failed ... see error above."; + } + else + { + // Wait for any recording delay + if (ofGetElapsedTimef() - delay_start > delay) + { + + // Write the capture to any built-in tracks + k4a_record_write_capture(recording, *capture); + + // Write the capture for any other custom tracks (not the IMU; do that after releasing the capture) + // ... + + if (recording_imu_enabled) + { + // Record IMU IMU data + record_imu(); + } + + // Indicate that we are recording + //cout << "."; + //cout.flush(); + + record_frame_count++; + } + else + { + // Print the countdown + // cout << (delay - (ofGetElapsedTimef() - delay_start)) << endl; + } + } + } + + void Record::record_imu() + { + // Code from k4arecorder (Line 213) + // https://github.com/microsoft/Azure-Kinect-Sensor-SDK/blob/master/tools/k4arecorder/recorder.cpp + // and https://docs.microsoft.com/en-us/azure/kinect-dk/retrieve-imu-samples + + k4a_wait_result_t result; + + // Loop to get the queued IMU samples after every capture. + // We kick out of the loop when result returns K4A_WAIT_RESULT_TIMEOUT + do + { + k4a_imu_sample_t sample; + result = k4a_device_get_imu_sample(device, &sample, 0); + if (result == K4A_WAIT_RESULT_SUCCEEDED) + { + // Write the IMU data to file + k4a_result_t write_result = k4a_record_write_imu_sample(recording, sample); + if (K4A_FAILED(write_result)) + { + ofLogError(__FUNCTION__) << "Runtime error: k4a_record_write_imu_sample() returned " << write_result; + break; + } + } + else if (result == K4A_WAIT_RESULT_TIMEOUT) + { + // Indicates that there are no queued samples and none have arrived in the timeout specified. + break; + } + else + { + ofLogError(__FUNCTION__) << "Runtime error: k4a_device_get_imu_sample() returned " << result; + break; + } + + // printf(" | Accelerometer temperature:%.2f x:%.4f y:%.4f z: %.4f\n", + // sample.temperature, + // sample.acc_sample.xyz.x, + // sample.acc_sample.xyz.y, + // sample.acc_sample.xyz.z); + + } while (result != K4A_WAIT_RESULT_FAILED); + } + + float Record::getTimerDelay() + { + return MAX(delay - (ofGetElapsedTimef() - delay_start), 0); + } + +} // namespace ofxAzureKinect \ No newline at end of file diff --git a/src/ofxAzureKinect/Record.h b/src/ofxAzureKinect/Record.h new file mode 100644 index 0000000..32b0a8c --- /dev/null +++ b/src/ofxAzureKinect/Record.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include + +#include "ofMain.h" + +namespace ofxAzureKinect +{ + class Record + { + + public: + Record(){} + ~Record(){} + + void setup(k4a_device_t device, k4a_device_configuration_t config, bool recording_imu_enabled = true, float delay = 3, string filename = ""); + void start(); + void stop(); + void record(k4a_capture_t *capture); + + // Set a recording delay (in seconds) + void setDelay(float _delay) { this->delay = _delay; } + float getTimerDelay(); + + uint32_t getRecordedFrameNum() const { + return record_frame_count; + } + private: + k4a_device_t device; + k4a_record_t recording; + + char *filename; + bool recording_imu_enabled = true; + + void record_imu(); + + bool k4a_failed = false; + + float delay = 0; + float delay_start = 0; + uint32_t record_frame_count = 0; + }; +} // namespace ofxAzureKinect \ No newline at end of file