kinect – プログラミング de 落描き

2018年9月22日2018年9月22日

Ascii man v2. openFrameworks

[ Video ]

[ About ]
Kinectから取得した画像と深度を使って特定の深さの箇所だけフォントでマスク

Send images and depth acquired from Kinect, replacement pictures to the shaders, switch pictures depicted in units of pixels

[ Source ]

#pragma once

#include <Windows.h>
#include <NuiApi.h>
#include <opencv2/opencv.hpp>
#include "ofMain.h"

class ofApp : public ofBaseApp {

public:
	void setup();
	void update();
	void draw();

	void keyPressed(int key) {}
	void keyReleased(int key) {}
	void mouseMoved(int x, int y) {}
	void mouseDragged(int x, int y, int button) {}
	void mousePressed(int x, int y, int button) {}
	void mouseReleased(int x, int y, int button) {}
	void mouseEntered(int x, int y) {}
	void mouseExited(int x, int y) {}
	void windowResized(int w, int h) {}
	void dragEvent(ofDragInfo dragInfo) {}
	void gotMessage(ofMessage msg) {}

	// Kinect
	INuiSensor* sensor;
	INuiCoordinateMapper* cordinate_mapper;
	HANDLE color_event;
	HANDLE color_handle;
	HANDLE depth_event;
	HANDLE depth_handle;
	HANDLE events[2];
	int width, height;

	// OpenCV
	cv::Mat frame;
	cv::Mat depth;

	// openFrameworks
	ofImage frame_img;
	ofImage depth_img;
	ofFbo fbo;
	ofShader shader;
	ofTrueTypeFont font;
};

#pragma once

#include <Windows.h>

#include <NuiApi.h>

#include <opencv2/opencv.hpp>

#include "ofMain.h"

class ofApp : public ofBaseApp {

public:

void setup();

void update();

void draw();

void keyPressed(int key) {}

void keyReleased(int key) {}

void mouseMoved(int x, int y) {}

void mouseDragged(int x, int y, int button) {}

void mousePressed(int x, int y, int button) {}

void mouseReleased(int x, int y, int button) {}

void mouseEntered(int x, int y) {}

void mouseExited(int x, int y) {}

void windowResized(int w, int h) {}

void dragEvent(ofDragInfo dragInfo) {}

void gotMessage(ofMessage msg) {}

// Kinect

INuiSensor* sensor;

INuiCoordinateMapper* cordinate_mapper;

HANDLE color_event;

HANDLE color_handle;

HANDLE depth_event;

HANDLE depth_handle;

HANDLE events[2];

int width, height;

// OpenCV

cv::Mat frame;

cv::Mat depth;

// openFrameworks

ofImage frame_img;

ofImage depth_img;

ofFbo fbo;

ofShader shader;

ofTrueTypeFont font;

};

#include "ofApp.h"

//--------------------------------------------------------------
void ofApp::setup() {

	ofSetFrameRate(30);
	ofSetWindowTitle("InstaKinect");

	// Create Instance
	this->sensor = nullptr;
	HRESULT result = S_OK;
	result = NuiCreateSensorByIndex(0, &sensor);
	if (FAILED(result)) {
		cout << "Error : Create Sensor" << endl;
		return;
	}

	// Initialize
	result = sensor->NuiInitialize(NUI_INITIALIZE_FLAG_USES_COLOR | NUI_INITIALIZE_FLAG_USES_DEPTH);
	if (FAILED(result)) {
		cout << "Error : Initialise COLOR" << endl;
		return;
	}

	// Color
	this->color_event = INVALID_HANDLE_VALUE;
	this->color_handle = INVALID_HANDLE_VALUE;
	color_event = CreateEventW(nullptr, true, false, nullptr);
	result = sensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, color_event, &color_handle);
	if (FAILED(result)) {
		cout << "Error : Color Stream Open" << endl;
		return;
	}

	// Depth
	this->depth_event = INVALID_HANDLE_VALUE;
	this->depth_handle = INVALID_HANDLE_VALUE;
	depth_event = CreateEventW(nullptr, true, false, nullptr);
	result = sensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_DEPTH, NUI_IMAGE_RESOLUTION_640x480, 0, 2, depth_event, &depth_handle);
	if (FAILED(result)) {
		cout << "Error : Depth Stream Open" << endl;
		return;
	}

	// Size
	unsigned long refWidth = 0;
	unsigned long refHeight = 0;
	NuiImageResolutionToSize(NUI_IMAGE_RESOLUTION_640x480, refWidth, refHeight);
	this->width = static_cast<int>(refWidth);
	this->height = static_cast<int>(refHeight);

	// Mapp
	this->cordinate_mapper = nullptr;
	result = sensor->NuiGetCoordinateMapper(&this->cordinate_mapper);
	if (FAILED(result)) {
		cout << "Error : Cordinate Mapper " << endl;
		return;
	}

	// OpenCV
	this->frame_img.allocate(this->width, this->height, OF_IMAGE_COLOR);
	this->frame = cv::Mat(this->frame_img.getHeight(), this->frame_img.getWidth(), CV_MAKETYPE(CV_8UC3, this->frame_img.getPixels().getNumChannels()), this->frame_img.getPixels().getData(), 0);
	this->depth_img.allocate(this->width, this->height, OF_IMAGE_COLOR);
	this->depth = cv::Mat(this->depth_img.getHeight(), this->depth_img.getWidth(), CV_MAKETYPE(CV_8UC3, this->depth_img.getPixels().getNumChannels()), this->depth_img.getPixels().getData(), 0);

	this->events[0] = this->color_event;
	this->events[1] = this->depth_event;

	string moji = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
	this->font.loadFont("fonts/Kazesawa-Bold.ttf", 13);

	this->fbo.allocate(ofGetWidth(), ofGetHeight());
	this->fbo.begin();
	ofClear(0);

	ofBackground(255);
	ofSetColor(0);

	for (int x = 0; x < ofGetWidth(); x += 10) {

		for (int y = 0; y < ofGetHeight(); y += 10) {

			this->font.drawString(moji.substr((int)ofRandom(26), 1), x, y);
		}
	}

	this->fbo.end();
	this->shader.load("", "shader/shader.frag");
}

//--------------------------------------------------------------
void ofApp::update() {

	HRESULT result = S_OK;

	std::vector<NUI_COLOR_IMAGE_POINT> color_point(width * height);

	// Wait frame update.
	ResetEvent(this->color_event);
	ResetEvent(this->depth_event);
	WaitForMultipleObjects(ARRAYSIZE(this->events), this->events, true, INFINITE);

	// Get frame from color camera.
	NUI_IMAGE_FRAME colorImageFrame = { 0 };
	result = sensor->NuiImageStreamGetNextFrame(this->color_handle, 0, &colorImageFrame);
	if (FAILED(result)) {
		std::cerr << "Error : NuiImageStreamGetNextFrame( COLOR )" << std::endl;
		return;
	}

	// Get image data.
	INuiFrameTexture* pColorFrameTexture = colorImageFrame.pFrameTexture;
	NUI_LOCKED_RECT colorLockedRect;
	pColorFrameTexture->LockRect(0, &colorLockedRect, nullptr, 0);

	//// Get frame from depth camera.
	NUI_IMAGE_FRAME depthImageFrame = { 0 };
	result = this->sensor->NuiImageStreamGetNextFrame(this->depth_handle, 0, &depthImageFrame);
	if (FAILED(result)) {
		std::cerr << "Error : NuiImageStreamGetNextFrame( DEPTH )" << std::endl;
		return;
	}

	// Get depth data.
	BOOL nearMode = false;
	INuiFrameTexture* pDepthFrameTexture = nullptr;
	this->sensor->NuiImageFrameGetDepthImagePixelFrameTexture(this->depth_handle, &depthImageFrame, &nearMode, &pDepthFrameTexture);
	NUI_LOCKED_RECT depthLockedRect;
	pDepthFrameTexture->LockRect(0, &depthLockedRect, nullptr, 0);

	// OpenCV(image)
	cv::Mat src = cv::Mat(this->height, this->width, CV_8UC4, reinterpret_cast<unsigned char*>(colorLockedRect.pBits));
	cv::cvtColor(src, this->frame, CV_BGRA2RGB);

	// OpenCV(depth)
	cv::Mat bufferMat = cv::Mat::zeros(height, width, CV_8UC3);
	NUI_DEPTH_IMAGE_PIXEL* depth_pixel = reinterpret_cast<NUI_DEPTH_IMAGE_PIXEL*>(depthLockedRect.pBits);
	this->cordinate_mapper->MapDepthFrameToColorFrame(NUI_IMAGE_RESOLUTION_640x480, width * height, depth_pixel, NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, width * height, &color_point[0]);
	for (int y = 0; y < height; y++) {
		for (int x = 0; x < width; x++) {
			unsigned int index = y * width + x;
			bufferMat.at<cv::Vec3b>(color_point[index].y, color_point[index].x)[0] = depth_pixel[index].depth / 255;
			bufferMat.at<cv::Vec3b>(color_point[index].y, color_point[index].x)[1] = depth_pixel[index].depth - (int)(depth_pixel[index].depth / 255) * 255;
		}
	}
	bufferMat.copyTo(this->depth);
	
	// Release
	pColorFrameTexture->UnlockRect(0);
	pDepthFrameTexture->UnlockRect(0);
	this->sensor->NuiImageStreamReleaseFrame(this->color_handle, &colorImageFrame);
	this->sensor->NuiImageStreamReleaseFrame(this->depth_handle, &depthImageFrame);

	// ofImage Update
	this->frame_img.update();
	this->depth_img.update();
}

//--------------------------------------------------------------
void ofApp::draw() {
	
	ofSetColor(255);

	this->shader.begin();
	this->shader.setUniform1f("time", ofGetElapsedTimef());
	this->shader.setUniform2f("resolution", ofGetWidth(), ofGetHeight());
	this->shader.setUniformTexture("tex1", this->frame_img, 1);
	this->shader.setUniformTexture("tex2", this->fbo.getTexture(), 2);
	this->shader.setUniformTexture("depth", this->depth_img, 3);

	ofRect(0, 0, ofGetWidth(), ofGetHeight());

	this->shader.end();
}

//========================================================================
int main() {
	ofSetupOpenGL(640, 480, OF_WINDOW);
	ofRunApp(new ofApp());
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

#include "ofApp.h"

//--------------------------------------------------------------

void ofApp::setup() {

ofSetFrameRate(30);

ofSetWindowTitle("InstaKinect");

// Create Instance

this->sensor = nullptr;

HRESULT result = S_OK;

result = NuiCreateSensorByIndex(0, &sensor);

if (FAILED(result)) {

cout << "Error : Create Sensor" << endl;

return;

}

// Initialize

result = sensor->NuiInitialize(NUI_INITIALIZE_FLAG_USES_COLOR | NUI_INITIALIZE_FLAG_USES_DEPTH);

if (FAILED(result)) {

cout << "Error : Initialise COLOR" << endl;

return;

}

// Color

this->color_event = INVALID_HANDLE_VALUE;

this->color_handle = INVALID_HANDLE_VALUE;

color_event = CreateEventW(nullptr, true, false, nullptr);

result = sensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, color_event, &color_handle);

if (FAILED(result)) {

cout << "Error : Color Stream Open" << endl;

return;

}

// Depth

this->depth_event = INVALID_HANDLE_VALUE;

this->depth_handle = INVALID_HANDLE_VALUE;

depth_event = CreateEventW(nullptr, true, false, nullptr);

result = sensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_DEPTH, NUI_IMAGE_RESOLUTION_640x480, 0, 2, depth_event, &depth_handle);

if (FAILED(result)) {

cout << "Error : Depth Stream Open" << endl;

return;

}

// Size

unsigned long refWidth = 0;

unsigned long refHeight = 0;

NuiImageResolutionToSize(NUI_IMAGE_RESOLUTION_640x480, refWidth, refHeight);

this->width = static_cast<int>(refWidth);

this->height = static_cast<int>(refHeight);

// Mapp

this->cordinate_mapper = nullptr;

result = sensor->NuiGetCoordinateMapper(&this->cordinate_mapper);

if (FAILED(result)) {

cout << "Error : Cordinate Mapper " << endl;

return;

}

// OpenCV

this->frame_img.allocate(this->width, this->height, OF_IMAGE_COLOR);

this->frame = cv::Mat(this->frame_img.getHeight(), this->frame_img.getWidth(), CV_MAKETYPE(CV_8UC3, this->frame_img.getPixels().getNumChannels()), this->frame_img.getPixels().getData(), 0);

this->depth_img.allocate(this->width, this->height, OF_IMAGE_COLOR);

this->depth = cv::Mat(this->depth_img.getHeight(), this->depth_img.getWidth(), CV_MAKETYPE(CV_8UC3, this->depth_img.getPixels().getNumChannels()), this->depth_img.getPixels().getData(), 0);

this->events[0] = this->color_event;

this->events[1] = this->depth_event;

string moji = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";

this->font.loadFont("fonts/Kazesawa-Bold.ttf", 13);

this->fbo.allocate(ofGetWidth(), ofGetHeight());

this->fbo.begin();

ofClear(0);

ofBackground(255);

ofSetColor(0);

for (int x = 0; x < ofGetWidth(); x += 10) {

for (int y = 0; y < ofGetHeight(); y += 10) {

this->font.drawString(moji.substr((int)ofRandom(26), 1), x, y);

}

this->fbo.end();

this->shader.load("", "shader/shader.frag");

}

//--------------------------------------------------------------

void ofApp::update() {

HRESULT result = S_OK;

std::vector<NUI_COLOR_IMAGE_POINT> color_point(width * height);

// Wait frame update.

ResetEvent(this->color_event);

ResetEvent(this->depth_event);

WaitForMultipleObjects(ARRAYSIZE(this->events), this->events, true, INFINITE);

// Get frame from color camera.

NUI_IMAGE_FRAME colorImageFrame = { 0 };

result = sensor->NuiImageStreamGetNextFrame(this->color_handle, 0, &colorImageFrame);

if (FAILED(result)) {

std::cerr << "Error : NuiImageStreamGetNextFrame( COLOR )" << std::endl;

return;

}

// Get image data.

INuiFrameTexture* pColorFrameTexture = colorImageFrame.pFrameTexture;

NUI_LOCKED_RECT colorLockedRect;

pColorFrameTexture->LockRect(0, &colorLockedRect, nullptr, 0);

//// Get frame from depth camera.

NUI_IMAGE_FRAME depthImageFrame = { 0 };

result = this->sensor->NuiImageStreamGetNextFrame(this->depth_handle, 0, &depthImageFrame);

if (FAILED(result)) {

std::cerr << "Error : NuiImageStreamGetNextFrame( DEPTH )" << std::endl;

return;

}

// Get depth data.

BOOL nearMode = false;

INuiFrameTexture* pDepthFrameTexture = nullptr;

this->sensor->NuiImageFrameGetDepthImagePixelFrameTexture(this->depth_handle, &depthImageFrame, &nearMode, &pDepthFrameTexture);

NUI_LOCKED_RECT depthLockedRect;

pDepthFrameTexture->LockRect(0, &depthLockedRect, nullptr, 0);

// OpenCV(image)

cv::Mat src = cv::Mat(this->height, this->width, CV_8UC4, reinterpret_cast<unsigned char*>(colorLockedRect.pBits));

cv::cvtColor(src, this->frame, CV_BGRA2RGB);

// OpenCV(depth)

cv::Mat bufferMat = cv::Mat::zeros(height, width, CV_8UC3);

NUI_DEPTH_IMAGE_PIXEL* depth_pixel = reinterpret_cast<NUI_DEPTH_IMAGE_PIXEL*>(depthLockedRect.pBits);

this->cordinate_mapper->MapDepthFrameToColorFrame(NUI_IMAGE_RESOLUTION_640x480, width * height, depth_pixel, NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, width * height, &color_point[0]);

for (int y = 0; y < height; y++) {

for (int x = 0; x < width; x++) {

unsigned int index = y * width + x;

bufferMat.at<cv::Vec3b>(color_point[index].y, color_point[index].x)[0] = depth_pixel[index].depth / 255;

bufferMat.at<cv::Vec3b>(color_point[index].y, color_point[index].x)[1] = depth_pixel[index].depth - (int)(depth_pixel[index].depth / 255) * 255;

}

bufferMat.copyTo(this->depth);

// Release

pColorFrameTexture->UnlockRect(0);

pDepthFrameTexture->UnlockRect(0);

this->sensor->NuiImageStreamReleaseFrame(this->color_handle, &colorImageFrame);

this->sensor->NuiImageStreamReleaseFrame(this->depth_handle, &depthImageFrame);

// ofImage Update

this->frame_img.update();

this->depth_img.update();

}

//--------------------------------------------------------------

void ofApp::draw() {

ofSetColor(255);

this->shader.begin();

this->shader.setUniform1f("time", ofGetElapsedTimef());

this->shader.setUniform2f("resolution", ofGetWidth(), ofGetHeight());

this->shader.setUniformTexture("tex1", this->frame_img, 1);

this->shader.setUniformTexture("tex2", this->fbo.getTexture(), 2);

this->shader.setUniformTexture("depth", this->depth_img, 3);

ofRect(0, 0, ofGetWidth(), ofGetHeight());

this->shader.end();

}

//========================================================================

int main() {

ofSetupOpenGL(640, 480, OF_WINDOW);

ofRunApp(new ofApp());

}

#version 150

uniform mat4 modelViewProjectionMatrix;
in vec4 position;

void main(){
	gl_Position = modelViewProjectionMatrix * position;
}

#version 150

uniform mat4 modelViewProjectionMatrix;

in vec4 position;

void main(){

gl_Position = modelViewProjectionMatrix * position;

}

#version 150

uniform float time;
uniform vec2 resolution;
uniform sampler2DRect tex1;
uniform sampler2DRect tex2;
uniform sampler2DRect depth;

out vec4 outputColor;

void main() {

  vec2 p = (gl_FragCoord.xy * 2.0 - resolution) / min(resolution.x, resolution.y);

  vec4 t1 = texture(tex1, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y));
  vec4 t2 = texture(tex2, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y));

  float r = texture(depth, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y)).r;
  float g = texture(depth, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y)).g;
  float d = r * 255.0 + g;
  d /= 65,536;

  vec4 color = d < 0.1 ? t2 + t1 : vec4(1.0);
  gl_FragColor = color;
}

#version 150

uniform float time;

uniform vec2 resolution;

uniform sampler2DRect tex1;

uniform sampler2DRect tex2;

uniform sampler2DRect depth;

out vec4 outputColor;

void main() {

vec2 p = (gl_FragCoord.xy * 2.0 - resolution) / min(resolution.x, resolution.y);

vec4 t1 = texture(tex1, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y));

vec4 t2 = texture(tex2, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y));

float r = texture(depth, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y)).r;

float g = texture(depth, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y)).g;

float d = r * 255.0 + g;

d /= 65,536;

vec4 color = d < 0.1 ? t2 + t1 : vec4(1.0);

gl_FragColor = color;

}

[ Link ]
https://github.com/junkiyoshi/Insta20180922

[ほしいものリスト]
Amazon

[ Sponsor Link ]

2018年9月17日

Ascii man. openframeworks

[ Video ]

[ About ]
Kinectから取得した画像&深度と差し替え用の絵をシェーダに送信、ピクセル単位の深度で描写する絵を切り替え。シェーダのおかげで描写がなめらか＆高速化！

Send images and depth acquired from Kinect, replacement pictures to the shaders, switch pictures depicted in units of pixels

[ Source ]

#pragma once

#include <Windows.h>
#include <NuiApi.h>
#include <opencv2/opencv.hpp>
#include "ofMain.h"

class ofApp : public ofBaseApp {

public:
	void setup();
	void update();
	void draw();

	void keyPressed(int key) {}
	void keyReleased(int key) {}
	void mouseMoved(int x, int y) {}
	void mouseDragged(int x, int y, int button) {}
	void mousePressed(int x, int y, int button) {}
	void mouseReleased(int x, int y, int button) {}
	void mouseEntered(int x, int y) {}
	void mouseExited(int x, int y) {}
	void windowResized(int w, int h) {}
	void dragEvent(ofDragInfo dragInfo) {}
	void gotMessage(ofMessage msg) {}

	// Kinect
	INuiSensor* sensor;
	INuiCoordinateMapper* cordinate_mapper;
	HANDLE color_event;
	HANDLE color_handle;
	HANDLE depth_event;
	HANDLE depth_handle;
	HANDLE events[2];
	int width, height;

	// OpenCV
	cv::Mat frame;
	cv::Mat depth;

	// openFrameworks
	ofImage frame_img;
	ofImage depth_img;
	ofFbo fbo;
	ofShader shader;
	ofTrueTypeFont font;
};

#pragma once

#include <Windows.h>

#include <NuiApi.h>

#include <opencv2/opencv.hpp>

#include "ofMain.h"

class ofApp : public ofBaseApp {

public:

void setup();

void update();

void draw();

void keyPressed(int key) {}

void keyReleased(int key) {}

void mouseMoved(int x, int y) {}

void mouseDragged(int x, int y, int button) {}

void mousePressed(int x, int y, int button) {}

void mouseReleased(int x, int y, int button) {}

void mouseEntered(int x, int y) {}

void mouseExited(int x, int y) {}

void windowResized(int w, int h) {}

void dragEvent(ofDragInfo dragInfo) {}

void gotMessage(ofMessage msg) {}

// Kinect

INuiSensor* sensor;

INuiCoordinateMapper* cordinate_mapper;

HANDLE color_event;

HANDLE color_handle;

HANDLE depth_event;

HANDLE depth_handle;

HANDLE events[2];

int width, height;

// OpenCV

cv::Mat frame;

cv::Mat depth;

// openFrameworks

ofImage frame_img;

ofImage depth_img;

ofFbo fbo;

ofShader shader;

ofTrueTypeFont font;

};

#include "ofApp.h"

//--------------------------------------------------------------
void ofApp::setup() {

	ofSetFrameRate(30);
	ofSetWindowTitle("InstaKinect");

	// Create Instance
	this->sensor = nullptr;
	HRESULT result = S_OK;
	result = NuiCreateSensorByIndex(0, &sensor);
	if (FAILED(result)) {
		cout << "Error : Create Sensor" << endl;
		return;
	}

	// Initialize
	result = sensor->NuiInitialize(NUI_INITIALIZE_FLAG_USES_COLOR | NUI_INITIALIZE_FLAG_USES_DEPTH);
	if (FAILED(result)) {
		cout << "Error : Initialise COLOR" << endl;
		return;
	}

	// Color
	this->color_event = INVALID_HANDLE_VALUE;
	this->color_handle = INVALID_HANDLE_VALUE;
	color_event = CreateEventW(nullptr, true, false, nullptr);
	result = sensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, color_event, &color_handle);
	if (FAILED(result)) {
		cout << "Error : Color Stream Open" << endl;
		return;
	}

	// Depth
	this->depth_event = INVALID_HANDLE_VALUE;
	this->depth_handle = INVALID_HANDLE_VALUE;
	depth_event = CreateEventW(nullptr, true, false, nullptr);
	result = sensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_DEPTH, NUI_IMAGE_RESOLUTION_640x480, 0, 2, depth_event, &depth_handle);
	if (FAILED(result)) {
		cout << "Error : Depth Stream Open" << endl;
		return;
	}

	// Size
	unsigned long refWidth = 0;
	unsigned long refHeight = 0;
	NuiImageResolutionToSize(NUI_IMAGE_RESOLUTION_640x480, refWidth, refHeight);
	this->width = static_cast<int>(refWidth);
	this->height = static_cast<int>(refHeight);

	// Mapp
	this->cordinate_mapper = nullptr;
	result = sensor->NuiGetCoordinateMapper(&this->cordinate_mapper);
	if (FAILED(result)) {
		cout << "Error : Cordinate Mapper " << endl;
		return;
	}

	// OpenCV
	this->frame_img.allocate(this->width, this->height, OF_IMAGE_COLOR);
	this->frame = cv::Mat(this->frame_img.getHeight(), this->frame_img.getWidth(), CV_MAKETYPE(CV_8UC3, this->frame_img.getPixels().getNumChannels()), this->frame_img.getPixels().getData(), 0);
	this->depth_img.allocate(this->width, this->height, OF_IMAGE_COLOR);
	this->depth = cv::Mat(this->depth_img.getHeight(), this->depth_img.getWidth(), CV_MAKETYPE(CV_8UC3, this->depth_img.getPixels().getNumChannels()), this->depth_img.getPixels().getData(), 0);

	this->events[0] = this->color_event;
	this->events[1] = this->depth_event;

	// FBO & Shader
	this->fbo.allocate(ofGetWidth(), ofGetHeight());
	this->shader.load("", "shader/shader.frag");

	this->font.loadFont("fonts/Kazesawa-Bold.ttf", 7);
}

//--------------------------------------------------------------
void ofApp::update() {

	HRESULT result = S_OK;

	// Color Table
	cv::Vec3b color[7];
	color[0] = cv::Vec3b(0, 0, 0);
	color[1] = cv::Vec3b(255, 0, 0);
	color[2] = cv::Vec3b(0, 255, 0);
	color[3] = cv::Vec3b(0, 0, 255);
	color[4] = cv::Vec3b(255, 255, 0);
	color[5] = cv::Vec3b(255, 0, 255);
	color[6] = cv::Vec3b(0, 255, 255);

	std::vector<NUI_COLOR_IMAGE_POINT> color_point(width * height);

	// Wait frame update.
	ResetEvent(this->color_event);
	ResetEvent(this->depth_event);
	WaitForMultipleObjects(ARRAYSIZE(this->events), this->events, true, INFINITE);

	// Get frame from color camera.
	NUI_IMAGE_FRAME colorImageFrame = { 0 };
	result = sensor->NuiImageStreamGetNextFrame(this->color_handle, 0, &colorImageFrame);
	if (FAILED(result)) {
		std::cerr << "Error : NuiImageStreamGetNextFrame( COLOR )" << std::endl;
		return;
	}

	// Get image data.
	INuiFrameTexture* pColorFrameTexture = colorImageFrame.pFrameTexture;
	NUI_LOCKED_RECT colorLockedRect;
	pColorFrameTexture->LockRect(0, &colorLockedRect, nullptr, 0);

	//// Get frame from depth camera.
	NUI_IMAGE_FRAME depthImageFrame = { 0 };
	result = this->sensor->NuiImageStreamGetNextFrame(this->depth_handle, 0, &depthImageFrame);
	if (FAILED(result)) {
		std::cerr << "Error : NuiImageStreamGetNextFrame( DEPTH )" << std::endl;
		return;
	}

	// Get depth data.
	BOOL nearMode = false;
	INuiFrameTexture* pDepthFrameTexture = nullptr;
	this->sensor->NuiImageFrameGetDepthImagePixelFrameTexture(this->depth_handle, &depthImageFrame, &nearMode, &pDepthFrameTexture);
	NUI_LOCKED_RECT depthLockedRect;
	pDepthFrameTexture->LockRect(0, &depthLockedRect, nullptr, 0);

	// OpenCV(image)
	cv::Mat src = cv::Mat(this->height, this->width, CV_8UC4, reinterpret_cast<unsigned char*>(colorLockedRect.pBits));
	cv::cvtColor(src, this->frame, CV_BGRA2RGB);

	// OpenCV(depth)
	cv::Mat bufferMat = cv::Mat::zeros(height, width, CV_8UC3);
	NUI_DEPTH_IMAGE_PIXEL* depth_pixel = reinterpret_cast<NUI_DEPTH_IMAGE_PIXEL*>(depthLockedRect.pBits);
	this->cordinate_mapper->MapDepthFrameToColorFrame(NUI_IMAGE_RESOLUTION_640x480, width * height, depth_pixel, NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, width * height, &color_point[0]);
	for (int y = 0; y < height; y++) {
		for (int x = 0; x < width; x++) {
			unsigned int index = y * width + x;
			bufferMat.at<cv::Vec3b>(color_point[index].y, color_point[index].x)[0] = depth_pixel[index].depth / 255;
			bufferMat.at<cv::Vec3b>(color_point[index].y, color_point[index].x)[1] = depth_pixel[index].depth - (int)(depth_pixel[index].depth / 255) * 255;
		}
	}
	bufferMat.copyTo(this->depth);
	
	// Release
	pColorFrameTexture->UnlockRect(0);
	pDepthFrameTexture->UnlockRect(0);
	this->sensor->NuiImageStreamReleaseFrame(this->color_handle, &colorImageFrame);
	this->sensor->NuiImageStreamReleaseFrame(this->depth_handle, &depthImageFrame);

	// ofImage Update
	this->frame_img.update();
	this->depth_img.update();

	// Make FBO
	ofSeedRandom(39);
	string moji = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
	this->fbo.begin();
	ofClear(0);

	ofBackground(0);

	for (int x = 0; x < ofGetWidth(); x += 10) {

		for (int y = 0; y < ofGetHeight(); y += 10) {

			ofSetColor(ofColor(0, 255 * ofNoise(x, y * 0.025 - ofGetFrameNum() * 0.05), 0));
			this->font.drawString(moji.substr((int)ofRandom(26), 1), x, y);
		}
	}

	this->fbo.end();

}

//--------------------------------------------------------------
void ofApp::draw() {

	ofSetColor(255);

	this->shader.begin();
	this->shader.setUniform1f("time", ofGetElapsedTimef());
	this->shader.setUniform2f("resolution", ofGetWidth(), ofGetHeight());
	this->shader.setUniformTexture("tex1", this->frame_img, 1);
	this->shader.setUniformTexture("tex2", this->fbo.getTexture(), 2);
	this->shader.setUniformTexture("depth", this->depth_img, 3);

	ofRect(0, 0, ofGetWidth(), ofGetHeight());

	this->shader.end();
}

//========================================================================
int main() {
	ofSetupOpenGL(640, 480, OF_WINDOW);
	ofRunApp(new ofApp());
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

#include "ofApp.h"

//--------------------------------------------------------------

void ofApp::setup() {

ofSetFrameRate(30);

ofSetWindowTitle("InstaKinect");

// Create Instance

this->sensor = nullptr;

HRESULT result = S_OK;

result = NuiCreateSensorByIndex(0, &sensor);

if (FAILED(result)) {

cout << "Error : Create Sensor" << endl;

return;

}

// Initialize

result = sensor->NuiInitialize(NUI_INITIALIZE_FLAG_USES_COLOR | NUI_INITIALIZE_FLAG_USES_DEPTH);

if (FAILED(result)) {

cout << "Error : Initialise COLOR" << endl;

return;

}

// Color

this->color_event = INVALID_HANDLE_VALUE;

this->color_handle = INVALID_HANDLE_VALUE;

color_event = CreateEventW(nullptr, true, false, nullptr);

result = sensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, color_event, &color_handle);

if (FAILED(result)) {

cout << "Error : Color Stream Open" << endl;

return;

}

// Depth

this->depth_event = INVALID_HANDLE_VALUE;

this->depth_handle = INVALID_HANDLE_VALUE;

depth_event = CreateEventW(nullptr, true, false, nullptr);

result = sensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_DEPTH, NUI_IMAGE_RESOLUTION_640x480, 0, 2, depth_event, &depth_handle);

if (FAILED(result)) {

cout << "Error : Depth Stream Open" << endl;

return;

}

// Size

unsigned long refWidth = 0;

unsigned long refHeight = 0;

NuiImageResolutionToSize(NUI_IMAGE_RESOLUTION_640x480, refWidth, refHeight);

this->width = static_cast<int>(refWidth);

this->height = static_cast<int>(refHeight);

// Mapp

this->cordinate_mapper = nullptr;

result = sensor->NuiGetCoordinateMapper(&this->cordinate_mapper);

if (FAILED(result)) {

cout << "Error : Cordinate Mapper " << endl;

return;

}

// OpenCV

this->frame_img.allocate(this->width, this->height, OF_IMAGE_COLOR);

this->frame = cv::Mat(this->frame_img.getHeight(), this->frame_img.getWidth(), CV_MAKETYPE(CV_8UC3, this->frame_img.getPixels().getNumChannels()), this->frame_img.getPixels().getData(), 0);

this->depth_img.allocate(this->width, this->height, OF_IMAGE_COLOR);

this->depth = cv::Mat(this->depth_img.getHeight(), this->depth_img.getWidth(), CV_MAKETYPE(CV_8UC3, this->depth_img.getPixels().getNumChannels()), this->depth_img.getPixels().getData(), 0);

this->events[0] = this->color_event;

this->events[1] = this->depth_event;

// FBO & Shader

this->fbo.allocate(ofGetWidth(), ofGetHeight());

this->shader.load("", "shader/shader.frag");

this->font.loadFont("fonts/Kazesawa-Bold.ttf", 7);

}

//--------------------------------------------------------------

void ofApp::update() {

HRESULT result = S_OK;

// Color Table

cv::Vec3b color[7];

color[0] = cv::Vec3b(0, 0, 0);

color[1] = cv::Vec3b(255, 0, 0);

color[2] = cv::Vec3b(0, 255, 0);

color[3] = cv::Vec3b(0, 0, 255);

color[4] = cv::Vec3b(255, 255, 0);

color[5] = cv::Vec3b(255, 0, 255);

color[6] = cv::Vec3b(0, 255, 255);

std::vector<NUI_COLOR_IMAGE_POINT> color_point(width * height);

// Wait frame update.

ResetEvent(this->color_event);

ResetEvent(this->depth_event);

WaitForMultipleObjects(ARRAYSIZE(this->events), this->events, true, INFINITE);

// Get frame from color camera.

NUI_IMAGE_FRAME colorImageFrame = { 0 };

result = sensor->NuiImageStreamGetNextFrame(this->color_handle, 0, &colorImageFrame);

if (FAILED(result)) {

std::cerr << "Error : NuiImageStreamGetNextFrame( COLOR )" << std::endl;

return;

}

// Get image data.

INuiFrameTexture* pColorFrameTexture = colorImageFrame.pFrameTexture;

NUI_LOCKED_RECT colorLockedRect;

pColorFrameTexture->LockRect(0, &colorLockedRect, nullptr, 0);

//// Get frame from depth camera.

NUI_IMAGE_FRAME depthImageFrame = { 0 };

result = this->sensor->NuiImageStreamGetNextFrame(this->depth_handle, 0, &depthImageFrame);

if (FAILED(result)) {

std::cerr << "Error : NuiImageStreamGetNextFrame( DEPTH )" << std::endl;

return;

}

// Get depth data.

BOOL nearMode = false;

INuiFrameTexture* pDepthFrameTexture = nullptr;

this->sensor->NuiImageFrameGetDepthImagePixelFrameTexture(this->depth_handle, &depthImageFrame, &nearMode, &pDepthFrameTexture);

NUI_LOCKED_RECT depthLockedRect;

pDepthFrameTexture->LockRect(0, &depthLockedRect, nullptr, 0);

// OpenCV(image)

cv::Mat src = cv::Mat(this->height, this->width, CV_8UC4, reinterpret_cast<unsigned char*>(colorLockedRect.pBits));

cv::cvtColor(src, this->frame, CV_BGRA2RGB);

// OpenCV(depth)

cv::Mat bufferMat = cv::Mat::zeros(height, width, CV_8UC3);

NUI_DEPTH_IMAGE_PIXEL* depth_pixel = reinterpret_cast<NUI_DEPTH_IMAGE_PIXEL*>(depthLockedRect.pBits);

this->cordinate_mapper->MapDepthFrameToColorFrame(NUI_IMAGE_RESOLUTION_640x480, width * height, depth_pixel, NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, width * height, &color_point[0]);

for (int y = 0; y < height; y++) {

for (int x = 0; x < width; x++) {

unsigned int index = y * width + x;

bufferMat.at<cv::Vec3b>(color_point[index].y, color_point[index].x)[0] = depth_pixel[index].depth / 255;

bufferMat.at<cv::Vec3b>(color_point[index].y, color_point[index].x)[1] = depth_pixel[index].depth - (int)(depth_pixel[index].depth / 255) * 255;

}

bufferMat.copyTo(this->depth);

// Release

pColorFrameTexture->UnlockRect(0);

pDepthFrameTexture->UnlockRect(0);

this->sensor->NuiImageStreamReleaseFrame(this->color_handle, &colorImageFrame);

this->sensor->NuiImageStreamReleaseFrame(this->depth_handle, &depthImageFrame);

// ofImage Update

this->frame_img.update();

this->depth_img.update();

// Make FBO

ofSeedRandom(39);

string moji = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";

this->fbo.begin();

ofClear(0);

ofBackground(0);

for (int x = 0; x < ofGetWidth(); x += 10) {

for (int y = 0; y < ofGetHeight(); y += 10) {

ofSetColor(ofColor(0, 255 * ofNoise(x, y * 0.025 - ofGetFrameNum() * 0.05), 0));

this->font.drawString(moji.substr((int)ofRandom(26), 1), x, y);

}

this->fbo.end();

}

//--------------------------------------------------------------

void ofApp::draw() {

ofSetColor(255);

this->shader.begin();

this->shader.setUniform1f("time", ofGetElapsedTimef());

this->shader.setUniform2f("resolution", ofGetWidth(), ofGetHeight());

this->shader.setUniformTexture("tex1", this->frame_img, 1);

this->shader.setUniformTexture("tex2", this->fbo.getTexture(), 2);

this->shader.setUniformTexture("depth", this->depth_img, 3);

ofRect(0, 0, ofGetWidth(), ofGetHeight());

this->shader.end();

}

//========================================================================

int main() {

ofSetupOpenGL(640, 480, OF_WINDOW);

ofRunApp(new ofApp());

}

#version 150

uniform float time;
uniform vec2 resolution;
uniform sampler2DRect tex1;
uniform sampler2DRect tex2;
uniform sampler2DRect depth;

out vec4 outputColor;

void main() {

  vec2 p = (gl_FragCoord.xy * 2.0 - resolution) / min(resolution.x, resolution.y);

  vec4 t1 = texture(tex1, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y));
  vec4 t2 = texture(tex2, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y));

  float r = texture(depth, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y)).r;
  float g = texture(depth, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y)).g;
  float d = r * 255.0 + g;
  d /= 65,536;

  vec4 color = d < 0.1 ? t2 : t1;
  gl_FragColor = color;
}

#version 150

uniform float time;

uniform vec2 resolution;

uniform sampler2DRect tex1;

uniform sampler2DRect tex2;

uniform sampler2DRect depth;

out vec4 outputColor;

void main() {

vec2 p = (gl_FragCoord.xy * 2.0 - resolution) / min(resolution.x, resolution.y);

vec4 t1 = texture(tex1, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y));

vec4 t2 = texture(tex2, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y));

float r = texture(depth, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y)).r;

float g = texture(depth, vec2(gl_FragCoord.x, resolution.y - gl_FragCoord.y)).g;

float d = r * 255.0 + g;

d /= 65,536;

vec4 color = d < 0.1 ? t2 : t1;

gl_FragColor = color;

}

[ Link ]
https://github.com/junkiyoshi/Insta20180917

[ほしいものリスト]
Amazon

[ Sponsor Link ]

2018年4月23日2018年4月22日

Kinect座標とNoise openFrameworks

[ Video ]

[ About ]
Kinectからとれる3次元座標にNoiseで位置づらしをしました。

[ Source ]

#pragma once

#include <Windows.h>
#include <NuiApi.h>
#include <opencv2/opencv.hpp>
#include "ofMain.h"

class ofApp : public ofBaseApp {

public:
	void setup();
	void update();
	void draw();

	void keyPressed(int key) {}
	void keyReleased(int key) {}
	void mouseMoved(int x, int y) {}
	void mouseDragged(int x, int y, int button) {}
	void mousePressed(int x, int y, int button) {}
	void mouseReleased(int x, int y, int button) {}
	void mouseEntered(int x, int y) {}
	void mouseExited(int x, int y) {}
	void windowResized(int w, int h) {}
	void dragEvent(ofDragInfo dragInfo) {}
	void gotMessage(ofMessage msg) {}

	// Kinect
	INuiSensor* sensor;
	INuiCoordinateMapper* cordinate_mapper;
	HANDLE color_event;
	HANDLE color_handle;
	HANDLE depth_event;
	HANDLE depth_handle;
	HANDLE events[2];
	int width, height;

	// OpenCV
	cv::Mat frame;
	cv::Mat depth;

	// openFrameworks
	ofEasyCam cam;
	ofLight light;
	ofImage frame_img;
};

#pragma once

#include <Windows.h>

#include <NuiApi.h>

#include <opencv2/opencv.hpp>

#include "ofMain.h"

class ofApp : public ofBaseApp {

public:

void setup();

void update();

void draw();

void keyPressed(int key) {}

void keyReleased(int key) {}

void mouseMoved(int x, int y) {}

void mouseDragged(int x, int y, int button) {}

void mousePressed(int x, int y, int button) {}

void mouseReleased(int x, int y, int button) {}

void mouseEntered(int x, int y) {}

void mouseExited(int x, int y) {}

void windowResized(int w, int h) {}

void dragEvent(ofDragInfo dragInfo) {}

void gotMessage(ofMessage msg) {}

// Kinect

INuiSensor* sensor;

INuiCoordinateMapper* cordinate_mapper;

HANDLE color_event;

HANDLE color_handle;

HANDLE depth_event;

HANDLE depth_handle;

HANDLE events[2];

int width, height;

// OpenCV

cv::Mat frame;

cv::Mat depth;

// openFrameworks

ofEasyCam cam;

ofLight light;

ofImage frame_img;

};

#include "ofApp.h"

//--------------------------------------------------------------
void ofApp::setup() {
	ofSetFrameRate(30);
	ofBackground(39);
	ofSetWindowTitle("InstaKinect");

	ofEnableDepthTest();
	ofSetColor(239);

	// Create Instance
	this->sensor = nullptr;
	HRESULT result = S_OK;
	result = NuiCreateSensorByIndex(0, &sensor);
	if (FAILED(result)) {
		cout << "Error : Create Sensor" << endl;
		return;
	}

	// Initialize
	result = sensor->NuiInitialize(NUI_INITIALIZE_FLAG_USES_COLOR | NUI_INITIALIZE_FLAG_USES_DEPTH);
	if (FAILED(result)) {
		cout << "Error : Initialise COLOR" << endl;
		return;
	}

	// Color
	this->color_event = INVALID_HANDLE_VALUE;
	this->color_handle = INVALID_HANDLE_VALUE;
	color_event = CreateEventW(nullptr, true, false, nullptr);
	result = sensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, color_event, &color_handle);
	if (FAILED(result)) {
		cout << "Error : Color Stream Open" << endl;
		return;
	}

	// Depth
	this->depth_event = INVALID_HANDLE_VALUE;
	this->depth_handle = INVALID_HANDLE_VALUE;
	depth_event = CreateEventW(nullptr, true, false, nullptr);
	result = sensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_DEPTH, NUI_IMAGE_RESOLUTION_640x480, 0, 2, depth_event, &depth_handle);
	if (FAILED(result)) {
		cout << "Error : Depth Stream Open" << endl;
		return;
	}

	// Size
	unsigned long refWidth = 0;
	unsigned long refHeight = 0;
	NuiImageResolutionToSize(NUI_IMAGE_RESOLUTION_640x480, refWidth, refHeight);
	this->width = static_cast<int>(refWidth);
	this->height = static_cast<int>(refHeight);

	// Mapp
	this->cordinate_mapper = nullptr;
	result = sensor->NuiGetCoordinateMapper(&this->cordinate_mapper);
	if (FAILED(result)) {
		cout << "Error : Cordinate Mapper " << endl;
		return;
	}

	// OpenCV
	this->frame_img.allocate(this->width, this->height, OF_IMAGE_COLOR);
	this->frame = cv::Mat(this->frame_img.getHeight(), this->frame_img.getWidth(), CV_MAKETYPE(CV_8UC3, this->frame_img.getPixels().getNumChannels()), this->frame_img.getPixels().getData(), 0);
	this->depth = cv::Mat(height, width, CV_16UC1);

	this->events[0] = this->color_event;
	this->events[1] = this->depth_event;

}

//--------------------------------------------------------------
void ofApp::update() {

	HRESULT result = S_OK;

	// Color Table
	cv::Vec3b color[7];
	color[0] = cv::Vec3b(0, 0, 0);
	color[1] = cv::Vec3b(255, 0, 0);
	color[2] = cv::Vec3b(0, 255, 0);
	color[3] = cv::Vec3b(0, 0, 255);
	color[4] = cv::Vec3b(255, 255, 0);
	color[5] = cv::Vec3b(255, 0, 255);
	color[6] = cv::Vec3b(0, 255, 255);

	std::vector<NUI_COLOR_IMAGE_POINT> color_point(width * height);

	// Wait frame update.
	ResetEvent(this->color_event);
	ResetEvent(this->depth_event);
	WaitForMultipleObjects(ARRAYSIZE(this->events), this->events, true, INFINITE);

	// Get frame from color camera.
	NUI_IMAGE_FRAME colorImageFrame = { 0 };
	result = sensor->NuiImageStreamGetNextFrame(this->color_handle, 0, &colorImageFrame);
	if (FAILED(result)) {
		std::cerr << "Error : NuiImageStreamGetNextFrame( COLOR )" << std::endl;
		return;
	}

	// Get image data.
	INuiFrameTexture* pColorFrameTexture = colorImageFrame.pFrameTexture;
	NUI_LOCKED_RECT colorLockedRect;
	pColorFrameTexture->LockRect(0, &colorLockedRect, nullptr, 0);

	//// Get frame from depth camera.
	NUI_IMAGE_FRAME depthImageFrame = { 0 };
	result = this->sensor->NuiImageStreamGetNextFrame(this->depth_handle, 0, &depthImageFrame);
	if (FAILED(result)) {
		std::cerr << "Error : NuiImageStreamGetNextFrame( DEPTH )" << std::endl;
		return;
	}

	// Get depth data.
	BOOL nearMode = false;
	INuiFrameTexture* pDepthFrameTexture = nullptr;
	this->sensor->NuiImageFrameGetDepthImagePixelFrameTexture(this->depth_handle, &depthImageFrame, &nearMode, &pDepthFrameTexture);
	NUI_LOCKED_RECT depthLockedRect;
	pDepthFrameTexture->LockRect(0, &depthLockedRect, nullptr, 0);

	// OpenCV(image)
	cv::Mat src = cv::Mat(this->height, this->width, CV_8UC4, reinterpret_cast<unsigned char*>(colorLockedRect.pBits));
	cv::cvtColor(src, this->frame, CV_BGRA2RGB);
	cv::imshow("frame", this->frame);

	// OpenCV(depth)
	cv::Mat bufferMat = cv::Mat::zeros(height, width, CV_16UC1);
	NUI_DEPTH_IMAGE_PIXEL* depth_pixel = reinterpret_cast<NUI_DEPTH_IMAGE_PIXEL*>(depthLockedRect.pBits);
	this->cordinate_mapper->MapDepthFrameToColorFrame(NUI_IMAGE_RESOLUTION_640x480, width * height, depth_pixel, NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, width * height, &color_point[0]);
	for (int y = 0; y < height; y++) {
		for (int x = 0; x < width; x++) {
			unsigned int index = y * width + x;
			bufferMat.at<unsigned short>(color_point[index].y, color_point[index].x) = depth_pixel[index].depth;
		}
	}
	bufferMat.copyTo(this->depth);
	//cv::imshow("depth", this->depth);

	// Release
	pColorFrameTexture->UnlockRect(0);
	pDepthFrameTexture->UnlockRect(0);
	this->sensor->NuiImageStreamReleaseFrame(this->color_handle, &colorImageFrame);
	this->sensor->NuiImageStreamReleaseFrame(this->depth_handle, &depthImageFrame);

	this->light.setPosition(ofVec3f(0, 0, 4096));
	ofEnableLighting();
	this->light.enable();
}

//--------------------------------------------------------------
void ofApp::draw() {

	this->cam.begin();
	ofTranslate(-ofGetWidth() / 2, -ofGetHeight() / 2, -512);
	ofRotateX(180);

	for (int y = 0; y < this->depth.rows; y += 5) {

		for (int x = 0; x < this->depth.cols; x += 5) {

			int z = depth.at<unsigned short>(y, x);

			if (z != 0 && z < 1024 + 512) {

				ofPushMatrix();
				ofTranslate(x, y, z);

				float noise_x = ofNoise(x * 0.005, ofGetFrameNum() * 0.05);
				float noise_y = ofNoise(y * 0.005, ofGetFrameNum() * 0.05);
				float noise_z = ofNoise(z * 0.005, ofGetFrameNum() * 0.05);
				float gap = 100;

				ofPoint noise_point = ofPoint(ofMap(noise_x, 0, 1, -gap, gap), ofMap(noise_y, 0, 1, -gap, gap), ofMap(noise_z, 0, 1, 0, -gap * 2));
				
				if (noise_point.length() < gap) {

					ofSetColor(ofMap(noise_x, 0, 1, 0, 255), ofMap(noise_y, 0, 1, 0, 255), ofMap(noise_z, 0, 1, 0, 255));
					ofDrawLine(ofPoint(), noise_point);
					ofDrawBox(noise_point, 8);
				}
				else {

					ofSetColor(239);
					ofDrawBox(ofPoint(), 8);
				}

				ofPopMatrix();
			}
		}
	}

	this->cam.end();
}

//========================================================================
int main() {
	ofSetupOpenGL(720, 720, OF_WINDOW);
	ofRunApp(new ofApp());
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

#include "ofApp.h"

//--------------------------------------------------------------

void ofApp::setup() {

ofSetFrameRate(30);

ofBackground(39);

ofSetWindowTitle("InstaKinect");

ofEnableDepthTest();

ofSetColor(239);

// Create Instance

this->sensor = nullptr;

HRESULT result = S_OK;

result = NuiCreateSensorByIndex(0, &sensor);

if (FAILED(result)) {

cout << "Error : Create Sensor" << endl;

return;

}

// Initialize

result = sensor->NuiInitialize(NUI_INITIALIZE_FLAG_USES_COLOR | NUI_INITIALIZE_FLAG_USES_DEPTH);

if (FAILED(result)) {

cout << "Error : Initialise COLOR" << endl;

return;

}

// Color

this->color_event = INVALID_HANDLE_VALUE;

this->color_handle = INVALID_HANDLE_VALUE;

color_event = CreateEventW(nullptr, true, false, nullptr);

result = sensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, color_event, &color_handle);

if (FAILED(result)) {

cout << "Error : Color Stream Open" << endl;

return;

}

// Depth

this->depth_event = INVALID_HANDLE_VALUE;

this->depth_handle = INVALID_HANDLE_VALUE;

depth_event = CreateEventW(nullptr, true, false, nullptr);

result = sensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_DEPTH, NUI_IMAGE_RESOLUTION_640x480, 0, 2, depth_event, &depth_handle);

if (FAILED(result)) {

cout << "Error : Depth Stream Open" << endl;

return;

}

// Size

unsigned long refWidth = 0;

unsigned long refHeight = 0;

NuiImageResolutionToSize(NUI_IMAGE_RESOLUTION_640x480, refWidth, refHeight);

this->width = static_cast<int>(refWidth);

this->height = static_cast<int>(refHeight);

// Mapp

this->cordinate_mapper = nullptr;

result = sensor->NuiGetCoordinateMapper(&this->cordinate_mapper);

if (FAILED(result)) {

cout << "Error : Cordinate Mapper " << endl;

return;

}

// OpenCV

this->frame_img.allocate(this->width, this->height, OF_IMAGE_COLOR);

this->frame = cv::Mat(this->frame_img.getHeight(), this->frame_img.getWidth(), CV_MAKETYPE(CV_8UC3, this->frame_img.getPixels().getNumChannels()), this->frame_img.getPixels().getData(), 0);

this->depth = cv::Mat(height, width, CV_16UC1);

this->events[0] = this->color_event;

this->events[1] = this->depth_event;

}

//--------------------------------------------------------------

void ofApp::update() {

HRESULT result = S_OK;

// Color Table

cv::Vec3b color[7];

color[0] = cv::Vec3b(0, 0, 0);

color[1] = cv::Vec3b(255, 0, 0);

color[2] = cv::Vec3b(0, 255, 0);

color[3] = cv::Vec3b(0, 0, 255);

color[4] = cv::Vec3b(255, 255, 0);

color[5] = cv::Vec3b(255, 0, 255);

color[6] = cv::Vec3b(0, 255, 255);

std::vector<NUI_COLOR_IMAGE_POINT> color_point(width * height);

// Wait frame update.

ResetEvent(this->color_event);

ResetEvent(this->depth_event);

WaitForMultipleObjects(ARRAYSIZE(this->events), this->events, true, INFINITE);

// Get frame from color camera.

NUI_IMAGE_FRAME colorImageFrame = { 0 };

result = sensor->NuiImageStreamGetNextFrame(this->color_handle, 0, &colorImageFrame);

if (FAILED(result)) {

std::cerr << "Error : NuiImageStreamGetNextFrame( COLOR )" << std::endl;

return;

}

// Get image data.

INuiFrameTexture* pColorFrameTexture = colorImageFrame.pFrameTexture;

NUI_LOCKED_RECT colorLockedRect;

pColorFrameTexture->LockRect(0, &colorLockedRect, nullptr, 0);

//// Get frame from depth camera.

NUI_IMAGE_FRAME depthImageFrame = { 0 };

result = this->sensor->NuiImageStreamGetNextFrame(this->depth_handle, 0, &depthImageFrame);

if (FAILED(result)) {

std::cerr << "Error : NuiImageStreamGetNextFrame( DEPTH )" << std::endl;

return;

}

// Get depth data.

BOOL nearMode = false;

INuiFrameTexture* pDepthFrameTexture = nullptr;

this->sensor->NuiImageFrameGetDepthImagePixelFrameTexture(this->depth_handle, &depthImageFrame, &nearMode, &pDepthFrameTexture);

NUI_LOCKED_RECT depthLockedRect;

pDepthFrameTexture->LockRect(0, &depthLockedRect, nullptr, 0);

// OpenCV(image)

cv::Mat src = cv::Mat(this->height, this->width, CV_8UC4, reinterpret_cast<unsigned char*>(colorLockedRect.pBits));

cv::cvtColor(src, this->frame, CV_BGRA2RGB);

cv::imshow("frame", this->frame);

// OpenCV(depth)

cv::Mat bufferMat = cv::Mat::zeros(height, width, CV_16UC1);

NUI_DEPTH_IMAGE_PIXEL* depth_pixel = reinterpret_cast<NUI_DEPTH_IMAGE_PIXEL*>(depthLockedRect.pBits);

this->cordinate_mapper->MapDepthFrameToColorFrame(NUI_IMAGE_RESOLUTION_640x480, width * height, depth_pixel, NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, width * height, &color_point[0]);

for (int y = 0; y < height; y++) {

for (int x = 0; x < width; x++) {

unsigned int index = y * width + x;

bufferMat.at<unsigned short>(color_point[index].y, color_point[index].x) = depth_pixel[index].depth;

}

bufferMat.copyTo(this->depth);

//cv::imshow("depth", this->depth);

// Release

pColorFrameTexture->UnlockRect(0);

pDepthFrameTexture->UnlockRect(0);

this->sensor->NuiImageStreamReleaseFrame(this->color_handle, &colorImageFrame);

this->sensor->NuiImageStreamReleaseFrame(this->depth_handle, &depthImageFrame);

this->light.setPosition(ofVec3f(0, 0, 4096));

ofEnableLighting();

this->light.enable();

}

//--------------------------------------------------------------

void ofApp::draw() {

this->cam.begin();

ofTranslate(-ofGetWidth() / 2, -ofGetHeight() / 2, -512);

ofRotateX(180);

for (int y = 0; y < this->depth.rows; y += 5) {

for (int x = 0; x < this->depth.cols; x += 5) {

int z = depth.at<unsigned short>(y, x);

if (z != 0 && z < 1024 + 512) {

ofPushMatrix();

ofTranslate(x, y, z);

float noise_x = ofNoise(x * 0.005, ofGetFrameNum() * 0.05);

float noise_y = ofNoise(y * 0.005, ofGetFrameNum() * 0.05);

float noise_z = ofNoise(z * 0.005, ofGetFrameNum() * 0.05);

float gap = 100;

ofPoint noise_point = ofPoint(ofMap(noise_x, 0, 1, -gap, gap), ofMap(noise_y, 0, 1, -gap, gap), ofMap(noise_z, 0, 1, 0, -gap * 2));

if (noise_point.length() < gap) {

ofSetColor(ofMap(noise_x, 0, 1, 0, 255), ofMap(noise_y, 0, 1, 0, 255), ofMap(noise_z, 0, 1, 0, 255));

ofDrawLine(ofPoint(), noise_point);

ofDrawBox(noise_point, 8);

}

else {

ofSetColor(239);

ofDrawBox(ofPoint(), 8);

}

ofPopMatrix();

}

this->cam.end();

}

//========================================================================

int main() {

ofSetupOpenGL(720, 720, OF_WINDOW);

ofRunApp(new ofApp());

}

[ Link ]
https://github.com/junkiyoshi/Insta20180422

[ほしいものリスト]
Amazon

[ Sponsor Link ]

2024年12月
月	火	水	木	金	土	日
						1
2	3	4	5	6	7	8
9	10	11	12	13	14	15
16	17	18	19	20	21	22
23	24	25	26	27	28	29
30	31