From 62a5aab5c29a75fa437dde00ca580ecdb3fb3e13 Mon Sep 17 00:00:00 2001 From: mareksuma1985 Date: Fri, 16 Jan 2015 20:06:29 +0100 Subject: [PATCH 1/4] added video4linux support http://pl.wikipedia.org/wiki/Video4Linux --- cvEyeTracker.c | 950 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 757 insertions(+), 193 deletions(-) diff --git a/cvEyeTracker.c b/cvEyeTracker.c index f468311..ef96af9 100644 --- a/cvEyeTracker.c +++ b/cvEyeTracker.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -43,18 +42,19 @@ #include #include #include -#include +#include /* file downloaded */ #include "remove_corneal_reflection.h" #include "ransac_ellipse.h" #include "timing.h" #include "svd.h" #ifdef _CH_ -#pragma package +#pragma package #endif #ifndef _EiC -#include "cv.h" +#include +/* fixed */ #include "highgui.h" #endif @@ -79,11 +79,40 @@ FILE *logfile; FILE *ellipse_log; -#define MIN_PUPIL_CONTOUR_POINTS 500 -#define MAX_PUPIL_CONTOUR_POINTS 10000 +#define MIN_PUPIL_CONTOUR_POINTS 500 +#define MAX_PUPIL_CONTOUR_POINTS 10000 #define PUPIL_SIZE_TOLERANCE 1000 //range of allowed pupil diameters #define MAX_CONTOUR_COUNT 20 +/* v4l capturer declarations part */ + +#include +#include /* getopt_long() */ +#include /* for videodev2.h */ +#include + +#define CLEAR(x) memset (&(x), 0, sizeof (x)) + +#define MAX_NORM 16 + +#ifndef MAX_INPUT +#define MAX_INPUT 16 +#endif + + char *dev_name = "/dev/video0"; + /* for additionalv4l camera */ +// char *dev_name2 = "/dev/video1"; + int fd = -1; +// int width = 640; +// int height = 480; + int n_buffers; + struct buffer *buffers = NULL; + //int index; + int cint; + int pixel_format = 1; + +/* end of v4l capturer declarations part */ + // Firewire Capture Variables int dev; int width=640,height=480,framerate=30; @@ -95,7 +124,8 @@ raw1394handle_t handle; nodeid_t * camera_nodes; dc1394_feature_set features; -// Load the source image. +// Load the source image. +IplImage *rgb_eye_image=NULL; /* needed only for long image conversion (YUYV > RGB > grayscale) */ IplImage *eye_image=NULL; IplImage *original_eye_image=NULL; IplImage *threshold_image=NULL; @@ -109,8 +139,8 @@ const char* ellipse_window = "Fitted Ellipse Window"; const char* scene_window = "Scene Image Window"; const char* control_window = "Parameter Control Window"; -char Feature_Names[9][30] ={ - "BRIGHTNESS", +char Feature_Names[9][30] = +{"BRIGHTNESS", "EXPOSURE", "SHARPNESS", "WHITE BALANCE", @@ -120,20 +150,20 @@ char Feature_Names[9][30] ={ "SHUTTER", "GAIN"}; -typedef struct { - int offset_value; +typedef struct +{ int offset_value; int value; int min; int max; - int available; - void (*callback)(int); -} camera_features; + int available; + void (*callback)(int);} +camera_features; camera_features eye_camera_features[9]; CvPoint pupil = {0,0}; //coordinates of pupil in tracker coordinate system CvPoint corneal_reflection = {0,0}; //coordinates of corneal reflection in tracker coordinate system -CvPoint diff_vector = {0,0}; //vector between the corneal reflection and pupil +CvPoint diff_vector = {0,0}; //vector between the corneal reflection and pupil int corneal_reflection_r = 0; //the radius of corneal reflection int view_cal_points = 1; @@ -143,21 +173,23 @@ int number_calibration_points_set = 0; int ok_calibrate = 0; CvPoint calipoints[CALIBRATIONPOINTS]; //conversion from eye to scene calibration points -CvPoint scenecalipoints[CALIBRATIONPOINTS]; //captured (with mouse) calibration points -CvPoint pucalipoints[CALIBRATIONPOINTS]; //captured eye points while looking at the calibration points in the scene -CvPoint crcalipoints[CALIBRATIONPOINTS]; //captured corneal reflection points while looking at the calibration points in the scene -CvPoint vectors[CALIBRATIONPOINTS]; //differences between the corneal reflection and pupil center +CvPoint scenecalipoints[CALIBRATIONPOINTS]; //captured (with mouse) calibration points +CvPoint pucalipoints[CALIBRATIONPOINTS]; //captured eye points while looking at the calibration points in the scene +CvPoint crcalipoints[CALIBRATIONPOINTS]; //captured corneal reflection points while looking at the calibration points in the scene +CvPoint vectors[CALIBRATIONPOINTS]; //differences between the corneal reflection and pupil center //scene coordinate interpolation variables float a, b, c, d, e; //temporary storage of coefficients -float aa, bb, cc, dd, ee; //pupil X coefficients -float ff, gg, hh, ii, jj; //pupil Y coefficients +float aa, bb, cc, dd, ee; //pupil X coefficients +float ff, gg, hh, ii, jj; //pupil Y coefficients float centx, centy; // translation to center pupil data after biquadratics -float cmx[4], cmy[4]; // corner correctioncoefficients +float cmx[4], cmy[4]; // corner correctioncoefficients int inx, iny; // translation to center pupil data before biquadratics -int White,Red,Green,Blue,Yellow; +CvScalar White, Red,Green,Blue,Yellow; +/* FIXED: +http://www.linuxforums.org/forum/programming-scripting/124273-cveyetracker-install-errors.html */ int frame_number=0; #define FRAMEW 640 @@ -174,8 +206,8 @@ double *avg_intensity_hori = (double*)malloc(FRAMEH*sizeof(double)); //horizonta //parameters for the algorithm int edge_threshold = 20; //threshold of pupil edge points detection -int rays = 18; //number of rays to use to detect feature points -int min_feature_candidates = 10; //minimum number of pupil feature candidates +int rays = 18; //number of rays to use to detect feature points +int min_feature_candidates = 10;//minimum number of pupil feature candidates int cr_window_size = 301; //corneal refelction search window size double map_matrix[3][3]; @@ -200,9 +232,12 @@ char ellipse_file[40]; #define FIX_UINT8(x) ( (x)<0 ? 0 : ((x)>255 ? 255:(x)) ) + //----------------------- Firewire Image Capture Code -----------------------// +//--------Uncomment if you have a properly installed firewire device.--------// -void Open_IEEE1394() +/* +void Open_IEEE1394() { int i; @@ -228,7 +263,7 @@ void Open_IEEE1394() dc1394_camera_on(handle, camera_nodes[i]); if (dc1394_dma_setup_capture(handle,camera_nodes[i], - i, /* channel */ + i, FORMAT_VGA_NONCOMPRESSED, cameramode[i], SPEED_400, @@ -239,7 +274,7 @@ void Open_IEEE1394() dc1394_destroy_handle(handle); exit(1); } - if (dc1394_start_iso_transmission(handle,cameras[i].node) !=DC1394_SUCCESS) { + if (dc1394_start_iso_transmission(handle,cameras[i].node) != DC1394_SUCCESS) { fprintf( stderr, "unable to start camera iso transmission\n"); dc1394_release_camera(handle,&cameras[i]); dc1394_destroy_handle(handle); @@ -249,14 +284,14 @@ void Open_IEEE1394() } } -void Grab_IEEE1394() +void Grab_IEEE1394() { if (dc1394_dma_multi_capture(cameras, numCameras)!=DC1394_SUCCESS) { fprintf( stderr, "unable to capture a frame\n"); } } -void Release_IEEE1394() +void Release_IEEE1394() { int i; @@ -265,7 +300,7 @@ void Release_IEEE1394() } } -void Close_IEEE1394() +void Close_IEEE1394() { int i; @@ -273,11 +308,314 @@ void Close_IEEE1394() if (dc1394_stop_iso_transmission(handle,cameras[i].node)!=DC1394_SUCCESS) { printf("couldn't stop the camera?\n"); } - dc1394_camera_off(handle, cameras[i].node); + dc1394_camera_off(handle, cameras[i].node); dc1394_dma_release_camera(handle,&cameras[i]); } dc1394_destroy_handle(handle); } +*/ + + +/*----------------------- V4L Image Capture Code -----------------------*/ +/* +This sample program was made by: + +Aquiles Yáñez C. +(yanezeloutfsmcl) + +Under the design guidance of: + +Agustín González V. + +version 0.1 - Lanzada en Enero del 2005 +version 0.2 - Lanzada en Febrero del 2005 +version 0.3 - Lanzada en Octubre del 2006 +version 0.4 - The same of 0.3 but in English (November 2009) +*/ + +//info needed to store one video frame in memory +struct buffer { + void * start; + size_t length; +}; + +static void errno_exit (const char *s) +{ + fprintf (stderr, "%s error %d, %s\n",s, errno, strerror (errno)); + exit (EXIT_FAILURE); +} + +//a blocking wrapper of the ioctl function +static int xioctl (int fd, int request, void *arg) +{ + int r; + + do r = ioctl (fd, request, arg); + while (-1 == r && EINTR == errno); + + return r; +} + +/* read one frame from memory */ +static int read_frame (int * fd, int width, int height, int * n_buffers, struct buffer * buffers, int pixel_format) +{ +struct v4l2_buffer buf; //needed for memory mapping +unsigned int i; +unsigned int Bpf; //bytes per frame + + if (read (*fd, buffers[0].start, buffers[0].length ) == -1) + { + switch (errno) + { + case EAGAIN: + return 0; + + case EIO: + //EIO ignored + default: + errno_exit ("read"); + } + } +/* writing to standard output */ +/* + switch (pixel_format) +{ + case 0: //YUV420 + Bpf = width*height*12/8; + break; + case 1: //YUYV + Bpf = width*height*2; + break; + case 2: //RGB565 + Bpf = width*height*2; + break; + case 3: //RGB32 + Bpf = width*height*4; + break; +} +*/ +/* int ret; + ret = write(STDOUT_FILENO, buffers[0].start, Bpf); + return 1; +*/ +} + + +//dummy function, that represents the stop of capturing +static void stop_capturing (int * fd) +{ + enum v4l2_buf_type type; + // Nothing to do. + +} + +//dummy function, that represents the start of capturing +static void start_capturing (int * fd, int * n_buffers ) +{ + unsigned int i; + enum v4l2_buf_type type; + // Nothing to do. + +} + +//allocate memory for buffers, the buffer must have capacity for one video frame. +static struct buffer *init_read (unsigned int buffer_size) +{ + struct buffer *buffers = NULL; + buffers = calloc (1, sizeof (*buffers)); + + if (!buffers) + { + fprintf (stderr, "Out of memory\n"); + exit (EXIT_FAILURE); + } + buffers[0].length = buffer_size; + buffers[0].start = malloc (buffer_size); + + if (!buffers[0].start) + { + fprintf (stderr, "Out of memory\n"); + exit (EXIT_FAILURE); + } + return buffers; +} + +//configure and initialize the hardware device +static struct buffer *init_device (int * fd, char * dev_name, int width, int height, int * n_buffers, int pixel_format) +{ + struct v4l2_capability cap; + struct v4l2_cropcap cropcap; + struct v4l2_crop crop; + struct v4l2_format fmt; + struct buffer * buffers = NULL; + unsigned int min; + + if (-1 == xioctl (*fd, VIDIOC_QUERYCAP, &cap)) + { + if (EINVAL == errno) + { + fprintf (stderr, "%s is no V4L2 device\n",dev_name); + exit (EXIT_FAILURE); + } else + { + errno_exit ("VIDIOC_QUERYCAP"); + } + } + + if (!(cap.capabilities & V4L2_CAP_VIDEO_CAPTURE)) + { + fprintf (stderr, "%s is not a video capture device\n",dev_name); + exit (EXIT_FAILURE); + } +/* + if (!(cap.capabilities & V4L2_CAP_READWRITE)) + { + fprintf (stderr, "%s does not support read i/o\n",dev_name); + exit (EXIT_FAILURE); + } +*/ + + //select video input, standard(not used) and tuner(not used) here + cropcap.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + if (-1 == xioctl (*fd, VIDIOC_CROPCAP, &cropcap)) + { + /* Errors ignored. */ + } + crop.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + crop.c = cropcap.defrect; /* reset to default */ + + if (-1 == xioctl (*fd, VIDIOC_S_CROP, &crop)) + { + switch (errno) + { + case EINVAL: + /* Cropping not supported. */ + break; + default: + /* Errors ignored. */ + break; + } + } + CLEAR (fmt); + //set image properties + fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + fmt.fmt.pix.width = width; + fmt.fmt.pix.height = height; + //fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUV420; + fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV; + //fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_RGB24; + //fmt.fmt.pix.colorspace = V4L2_COLORSPACE_SRGB; + + /* uncomment following line if you're using source of interlaced video */ + // fmt.fmt.pix.field = V4L2_FIELD_INTERLACED; + /* zmiana */ + + switch (pixel_format) + { + case 0: + fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUV420; + printf("Pixel format is YUV420"); + break; + case 1: + fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV; // YUYV or YUY2 or YUV422 (PS3 Eye image format) + printf("Pixel format is YUYV (YUY2, YUV422)"); + break; + case 2: + fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_RGB565; + printf("Pixel format is RGB565"); + break; + case 3: + fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_RGB32; + printf("Pixel format is RGB32"); + break; + } + + + if (-1 == xioctl (*fd, VIDIOC_S_FMT, &fmt)) + errno_exit ("VIDIOC_S_FMT"); + + /* Note VIDIOC_S_FMT may change width and height. */ + + //check the configuration data + min = fmt.fmt.pix.width * 2; + if (fmt.fmt.pix.bytesperline < min) + fmt.fmt.pix.bytesperline = min; + min = fmt.fmt.pix.bytesperline * fmt.fmt.pix.height; + if (fmt.fmt.pix.sizeimage < min) + fmt.fmt.pix.sizeimage = min; + + fprintf(stderr,"Video line size is %d Bytes\n", fmt.fmt.pix.bytesperline); + fprintf(stderr,"Video image size is %d Bytes\n", fmt.fmt.pix.sizeimage); +// fprintf(stderr,"Using READ IO Method\n"); + buffers=init_read (fmt.fmt.pix.sizeimage); + + return buffers; +} + +static void close_device (int * fd) +{ + if (-1 == close (*fd)) + errno_exit ("close"); + + *fd = -1; +} + +//free the buffers +static void uninit_device (int * n_buffers, struct buffer * buffers) +{ + unsigned int i; + + free (buffers[0].start); + free (buffers); +} + +static void open_device (int * fd, char * dev_name) +{ + struct stat st; + + if (-1 == stat (dev_name, &st)) + { + fprintf (stderr, "Cannot identify '%s': %d, %s\n", dev_name, errno, strerror (errno)); + exit (EXIT_FAILURE); + } + + if (!S_ISCHR (st.st_mode)) + { + fprintf (stderr, "%s is no device\n", dev_name); + exit (EXIT_FAILURE); + } + + *fd = open (dev_name, O_RDWR /* required */ | O_NONBLOCK, 0); + + if (-1 == *fd) + { + fprintf (stderr, "Cannot open for reading and writing '%s': %d, %s\n", dev_name, errno, strerror (errno)); + /* experimental */ + *fd = open (dev_name, O_RDONLY | O_NONBLOCK, 0); + if (-1 == *fd) + { + fprintf (stderr, "Cannot open for reading only'%s': %d, %s\n", dev_name, errno, strerror (errno)); + exit (EXIT_FAILURE); + } + + + exit (EXIT_FAILURE); + + } +} + + +void Release_v4ldevices() +{ +stop_capturing (&fd); +} +void Close_v4ldevices() +{ +close_device (&fd); +uninit_device (&n_buffers, buffers); +/* ADD code for additional camera here */ +} + //------------ map pupil coordinates to screen coordinates ---------/ CvPoint homography_map_point(CvPoint p) @@ -331,10 +669,10 @@ int cal_calibration_homography(void) int i, j; stuDPoint cal_scene[9], cal_eye[9]; stuDPoint scene_center, eye_center, *eye_nor, *scene_nor; - double dis_scale_scene, dis_scale_eye; + double dis_scale_scene, dis_scale_eye; for (i = 0; i < 9; i++) { - cal_scene[i].x = scenecalipoints[i].x; + cal_scene[i].x = scenecalipoints[i].x; cal_scene[i].y = scenecalipoints[i].y; cal_eye[i].x = vectors[i].x; cal_eye[i].y = vectors[i].y; @@ -392,8 +730,9 @@ int cal_calibration_homography(void) } for (i = 0; i < N; i++) { - map_matrix[i/3][i%3] = ppv[i][min_d_index]; //the column of v that corresponds to the smallest singular value, - //which is the solution of the equations + map_matrix[i/3][i%3] = ppv[i][min_d_index]; + /* the column of v that corresponds to the smallest + singular value, which is the solution of the equations */ } double T[3][3] = {0}, T1[3][3] = {0}; @@ -403,7 +742,7 @@ int cal_calibration_homography(void) printf("%8lf ", T1[j][i]); } printf("\n"); - } + } T[0][0] = T[1][1] = dis_scale_eye; T[0][2] = -dis_scale_eye*eye_center.x; @@ -416,16 +755,16 @@ int cal_calibration_homography(void) printf("%8lf ", map_matrix[j][i]); } printf("\n"); - } + } printf("\nT: \n"); for (j = 0; j < 3; j++) { for (i = 0; i < 3; i++) { printf("%8lf ", T[j][i]); } printf("\n"); - } + } - matrix_multiply33(map_matrix, T, map_matrix); + matrix_multiply33(map_matrix, T, map_matrix); T[0][0] = T[1][1] = dis_scale_scene; T[0][2] = -dis_scale_scene*scene_center.x; @@ -438,14 +777,14 @@ int cal_calibration_homography(void) printf("%8lf ", map_matrix[j][i]); } printf("\n"); - } + } printf("\nT: \n"); for (j = 0; j < 3; j++) { for (i = 0; i < 3; i++) { printf("%8lf ", T[j][i]); } printf("\n"); - } + } affine_matrix_inverse(T, T1); matrix_multiply33(T1, map_matrix, map_matrix); @@ -456,7 +795,7 @@ int cal_calibration_homography(void) printf("%8lf ", map_matrix[j][i]); } printf("\n"); - } + } for (i = 0; i < M; i++) { free(ppu[i]); @@ -467,83 +806,81 @@ int cal_calibration_homography(void) free(ppu); free(ppv); free(ppa); - + free(eye_nor); free(scene_nor); printf("\nfinish calculate calibration\n"); } -CvPoint map_point(CvPoint p) +CvPoint map_point(CvPoint p) { CvPoint p2; int quad=0; float x1,y1,xx,yy; // correct eye position by recentering offset: - x1 = (float) p.x; + x1 = (float) p.x; y1 = (float) p.y; // translate before biquadratic: - x1 -= inx; + x1 -= inx; y1 -= iny; // biquadratic mapping: - xx = aa+bb*x1+cc*y1+dd*x1*x1+ee*y1*y1; + xx = aa+bb*x1+cc*y1+dd*x1*x1+ee*y1*y1; yy = ff+gg*x1+hh*y1+ii*x1*x1+jj*y1*y1; // translate after biquadratic: - x1 = xx - centx; + x1 = xx - centx; y1 = yy - centy; // determine quadrant of point: - if (( x1<0 )&&( y1<0 )) quad = 0; + if (( x1<0 )&&( y1<0 )) quad = 0; else if (( x1>0 )&&( y1<0 )) quad = 1; else if (( x1<0 )&&( y1>0 )) quad = 2; else if (( x1>0 )&&( y1>0 )) quad = 3; // fix up by quadrant: - p2.x = (int)(xx + x1*y1*cmx[quad]); + p2.x = (int)(xx + x1*y1*cmx[quad]); p2.y = (int)(yy + x1*y1*cmy[quad]); return p2; } - - -//---------- calibration coefficient calculation ---------------// -// biquadratic equation fitter -// x, y are coordinates of eye tracker point -// X is x or y coordinate of screen point -// computes a, b, c, d, and e in the biquadratic -// X = a + b*(x-inx) + c*(y-iny) + d*(x-inx)*(x-inx) + e*(y-iny)*(y-iny) -// where inx = x1, y1 = y1 to reduce the solution to a 4x4 matrix - -void dqfit( float x1, float y1, - float x2, float y2, - float x3, float y3, - float x4, float y4, +//-------------- calibration coefficient calculation ---------------// +// biquadratic equation fitter +// x, y are coordinates of eye tracker point +// X is x or y coordinate of screen point +// computes a, b, c, d, and e in the biquadratic +// X = a + b*(x-inx) + c*(y-iny) + d*(x-inx)*(x-inx) + e*(y-iny)*(y-iny) +// where inx = x1, y1 = y1 to reduce the solution to a 4x4 matrix + +void dqfit( float x1, float y1, + float x2, float y2, + float x3, float y3, + float x4, float y4, float x5, float y5, float X1, float X2, float X3, float X4, float X5 ) { float den; - float x22,x32,x42,x52; // squared terms + float x22,x32,x42,x52; // squared terms float y22,y32,y42,y52; - inx = (int)x1; // record eye tracker centering constants + inx = (int)x1; // record eye tracker centering constants iny = (int)y1; - a = X1; // first coefficient - X2 -= X1; X3 -= X1; // center screen points + a = X1; // first coefficient + X2 -= X1; X3 -= X1; // center screen points X4 -= X1; X5 -= X1; - x2 -= x1; x3 -= x1; // center eye tracker points + x2 -= x1; x3 -= x1; // center eye tracker points x4 -= x1; x5 -= x1; - y2 -= y1; y3 -= y1; + y2 -= y1; y3 -= y1; y4 -= y1; y5 -= y1; - x22 = x2*x2; x32 = x3*x3; // squared terms of biquadratic + x22 = x2*x2; x32 = x3*x3; // squared terms of biquadratic x42 = x4*x4; x52 = x5*x5; y22 = y2*y2; y32 = y3*y3; y42 = y4*y4; y52 = y5*y5; - //Cramer's rule solution of 4x4 matrix */ +// Cramer's rule solution of 4x4 matrix */ den = -x2*y3*x52*y42-x22*y3*x4*y52+x22*y5*x4*y32-y22*x42*y3*x5- x32*y22*x4*y5-x42*x2*y5*y32+x32*x2*y5*y42-y2*x52*x4*y32+ x52*x2*y4*y32+y22*x52*y3*x4+y2*x42*x5*y32+x22*y3*x5*y42- @@ -584,7 +921,7 @@ void dqfit( float x1, float y1, int CalculateCalibration(void) { int i, j; - float x, y, wx[9], wy[9]; //work data points + float x, y, wx[9], wy[9]; //work data points int calx[10], caly[10]; //scene coordinate interpolation variables int eye_x[10], eye_y[10]; //scene coordinate interpolation variables @@ -603,7 +940,7 @@ int CalculateCalibration(void) } // Solve X biquadratic - dqfit((float)eye_x[0],(float)eye_y[0],(float)eye_x[1],(float)eye_y[1],(float)eye_x[2], + dqfit((float)eye_x[0],(float)eye_y[0],(float)eye_x[1],(float)eye_y[1],(float)eye_x[2], (float)eye_y[2],(float)eye_x[3],(float)eye_y[3],(float)eye_x[4],(float)eye_y[4], (float)calx[0],(float)calx[1],(float)calx[2],(float)calx[3],(float)calx[4]); aa = a; bb = b; cc = c; dd = d; ee = e; @@ -621,17 +958,17 @@ int CalculateCalibration(void) wx[i] = aa+bb*x+cc*y+dd*x*x+ee*y*y; wy[i] = ff+gg*x+hh*y+ii*x*x+jj*y*y; } - + // Shift screen points to center for quadrant compute - centx = wx[0]; + centx = wx[0]; centy = wy[0]; - + // Normalize to center: for(i = 0; i < 9; i++) { wx[i] -= centx; wy[i] -= centy; } - + // Compute coefficents for each quadrant for(i = 0; i < 4; i++) { j = i + 5; @@ -643,7 +980,7 @@ int CalculateCalibration(void) } -void Draw_Cross(IplImage *image, int centerx, int centery, int x_cross_length, int y_cross_length, double color) +void Draw_Cross(IplImage *image, int centerx, int centery, int x_cross_length, int y_cross_length, CvScalar color) { CvPoint pt1,pt2,pt3,pt4; @@ -663,26 +1000,26 @@ void Draw_Cross(IplImage *image, int centerx, int centery, int x_cross_length, i void Show_Calibration_Points() { - int i; - for (i=0;iimageData; int x,y; @@ -863,7 +1202,7 @@ void Normalize_Line_Histogram(IplImage *in_image) /* char adjustment; for (y=0;y> 1 )-1; @@ -933,7 +1272,8 @@ void uyyvyy2rgb (unsigned char *src, unsigned char *dest, unsigned long long int register int y0, y1, y2, y3, u, v; register int r, g, b; - while (i > 0) { + while (i > 0) + { y3 = (unsigned char) src[i--]; y2 = (unsigned char) src[i--]; v = (unsigned char) src[i--] - 128; @@ -964,23 +1304,215 @@ void FirewireFrame_to_RGBIplImage(void *FirewireFrame, IplImage *OpenCV_image) uyyvyy2rgb((unsigned char *)FirewireFrame, (unsigned char *)OpenCV_image->imageData, 640*480); } -void Grab_Camera_Frames() +//---------------------------- yuyv2 (YUYV, YUV422) to rgb24 ---------------------------------// +/* +This produces distorted colors, needs some tuning +*/ + +/* +#define YUYV2RGB(y, u, v, r, g, b)\ + u -= 128;\ + v -= 128;\ + r = y + ((v*1436) >> 10);\ + g = y - ((731 * v - 352* u) >> 10);\ + b = y + ((1814 * u) >> 10);\ +r= r < 0 ? 0 : r;\ +r= r > 255 ? 255 : r;\ +g= g < 0 ? 0 : g;\ +g= g > 255 ? 255 : g;\ +b= g < 0 ? 0 : b;\ +b= b > 255 ? 255 : b;\ + + +void yuyv2rgb (unsigned char *src, unsigned char *dest, unsigned long long int NumPixels) { - Grab_IEEE1394(); + register int i = NumPixels + ( NumPixels >> 1 )-1; + register int j = NumPixels + ( NumPixels << 1 )-1; + register int y1, y2, u, v; + register int r, g, b; - memcpy(eye_image->imageData,(char *)cameras[0].capture_buffer,monobytesperimage); - //memcpy(scene_image->imageData,(char *)cameras[1].capture_buffer,monobytesperimage); - FirewireFrame_to_RGBIplImage((unsigned char *)cameras[1].capture_buffer, scene_image); + while (i > 0) + { + u = (unsigned char) src[i--]; + y1 = (unsigned char) src[i--]; + v = (unsigned char) src[i--]; + y2 = (unsigned char) src[i--]; - original_eye_image = cvCloneImage(eye_image); + YUYV2RGB(y1, u, v, r, g, b); + dest[j--] = r; + dest[j--] = g; + dest[j--] = b; + YUYV2RGB(y2, u, v, r, g, b); + dest[j--] = r; + dest[j--] = g; + dest[j--] = b; + } +} +*/ + +/* +Written by Paul Bourke +August 2004 +http://paulbourke.net/ +*/ + +/* for transfering pixel values between functions */ +typedef struct +{ + char red; + char green; + char blue; +//char alpha; +} artifact; +/* artifact is the name of structure, pixel will be the name of the variable */ + +artifact YUV_to_Bitmap(int y,int u,int v) +{ + int r,g,b; + artifact pixel; + + pixel.red = 0; + pixel.green = 0; + pixel.blue = 0; +// pixel.alpha = 0; + + // u and v are +-0.5 + u -= 128; + v -= 128; + + // Conversion + r = y + 1.370705 * v; + g = y - 0.698001 * v - 0.337633 * u; + b = y + 1.732446 * u; +/* + r = y + 1.402 * v; + g = y - 0.344 * u - 0.714 * v; + b = y + 1.772 * u; +*/ + +/* + y -= 16; + r = 1.164 * y + 1.596 * v; + g = 1.164 * y - 0.392 * u - 0.813 * v; + b = 1.164 * y + 2.017 * u; +*/ + // Clamp to <0, 1> range + if (r < 0) r = 0; + if (g < 0) g = 0; + if (b < 0) b = 0; + if (r > 255) r = 255; + if (g > 255) g = 255; + if (b > 255) b = 255; + + pixel.red = r; + pixel.green = g; + pixel.blue = b; +// pixel.alpha = 0; + + return(pixel); +} + +void yuyv2rgb (unsigned char *src, unsigned char *dest, unsigned long long int NumPixels) +{ +artifact rgb1; +artifact rgb2; + /* Poprawic bo nieelegancko */ + // register int i = (NumPixels + ( NumPixels >> 1 ))*4/3-1; + /* Juz lepiej */ + register int i = (NumPixels << 1 )-1; /* 614400 Bytes in 640×480 YUYV frame */ + register int j = NumPixels + ( NumPixels << 1 )-1; /* 921600 Bytes in 640×480 RGB frame */ + int y1, y2, u, v; + int r, g, b; + + while (i > 0) + { + u = (unsigned char) src[i--]; + y1 = (unsigned char) src[i--]; + v = (unsigned char) src[i--]; + y2 = (unsigned char) src[i--]; + + rgb1 = YUV_to_Bitmap(y1,u,v); + dest[j--] = rgb1.red; + dest[j--] = rgb1.green; + dest[j--] = rgb1.blue; + rgb2 = YUV_to_Bitmap(y2,u,v); + dest[j--] = rgb2.red; + dest[j--] = rgb2.green; + dest[j--] = rgb2.blue; + } +} + +void v4lFrame_to_RGBIplImage(void *FirewireFrame, IplImage *OpenCV_image) +{ + yuyv2rgb((unsigned char *)FirewireFrame, (unsigned char *)OpenCV_image->imageData, 640*480); +} + +//---------------------------- yuyv2 (YUYV, YUV422) to grayscale ---------------------------------// +void yuyv2grey(unsigned char *src, /*IplImage *OpenCV_image*/unsigned char *dest, unsigned long long int NumPixels) +{ + register int i = (NumPixels <<1)-1; /* 614400 Bytes in 640×480 YUYV frame */ + register int j = NumPixels-1; /* 307200 Bytes in 640×480 grayscale frame */ + + while (i > 0) + { + /*u - we are skipping this byte */ src[i--]; + /*y1 - we are copying this byte */ dest[j--] = (unsigned char) src[i--]; + /*v - we are skipping this byte */ src[i--]; + /*y2 - we are copying this byte */ dest[j--] = (unsigned char) src[i--]; + } +} +/* POLACZYC te dwie funkcje! */ +void v4lFrame_to_grayscaleIplImage(void *FirewireFrame, IplImage *OpenCV_image) +{ + yuyv2grey((unsigned char *)FirewireFrame, (unsigned char *)OpenCV_image->imageData, 640*480); +} + +void Grab_Camera_Frames() +{ +/* for firewire cameras */ +//Grab_IEEE1394(); + /* for single firewire camera */ + //memcpy(eye_image->imageData,(char *)cameras[0].capture_buffer, monobytesperimage); + +/* for v4l cameras */ +read_frame (&fd, 640, 480, &n_buffers, buffers, 1); + + /* the slower way */ + /* copies RGB image from buffer */ + v4lFrame_to_RGBIplImage((unsigned char *)buffers[0].start, rgb_eye_image); + /* converts RGB image (3 Bytes per pixel) to grayscale image (1 Byte per pixel) */ + cvCvtColor(rgb_eye_image, eye_image, CV_RGB2GRAY); + original_eye_image = cvCloneImage(rgb_eye_image); + /* original_eye_image is color this way */ + + /* the faster way */ + /* when using a camera with filter blocking visible light, the image is already pretty much grayscale */ + /* copies only luminance information from YUYV image (2 Bytes per pixel) to grayscale image (1 Byte per pixel) */ + //v4lFrame_to_grayscaleIplImage((unsigned char *)buffers[0].start, eye_image); + //original_eye_image = cvCloneImage(eye_image); + /* original_eye_image is grayscale this way but who cares */ + +/* for additional firewire camera */ +//memcpy(scene_image->imageData,(char *)cameras[1].capture_buffer, monobytesperimage); +/* or */ +//FirewireFrame_to_RGBIplImage((unsigned char *)cameras[1].capture_buffer, scene_image); + +/* for additional v4l camera */ +/* replace first parameter of this function with your additional camera buffer */ +//v4lFrame_to_RGBIplImage((unsigned char *)buffers[1].start, scene_image); if (frame_number == 0) { Calculate_Avg_Intensity_Hori(eye_image); - memcpy(intensity_factor_hori, avg_intensity_hori, eye_image->height*sizeof(double)); - } + memcpy(intensity_factor_hori, avg_intensity_hori, eye_image->height*sizeof(double)); + } + +/* for firewire cameras */ +// Release_IEEE1394(); - Release_IEEE1394(); - frame_number++; +/* for v4l cameras */ +Release_v4ldevices(); + +frame_number++; } void process_image() @@ -994,25 +1526,25 @@ void process_image() cvZero(ellipse_image); cvSmooth(eye_image, eye_image, CV_GAUSSIAN, 5, 5); - Reduce_Line_Noise(eye_image); - + Reduce_Line_Noise(eye_image); + if (save_image == 1) { printf("save image %d\n", image_no); sprintf(eye_file, "./Eye/Eye_%05d.jpg", image_no); image_no++; cvSaveImage(eye_file, eye_image); } - + //corneal reflection - remove_corneal_reflection(eye_image, threshold_image, (int)start_point.x, (int)start_point.y, cr_window_size, - (int)eye_image->height/10, corneal_reflection.x, corneal_reflection.y, corneal_reflection_r); + remove_corneal_reflection(eye_image, threshold_image, (int)start_point.x, (int)start_point.y, cr_window_size, + (int)eye_image->height/10, corneal_reflection.x, corneal_reflection.y, corneal_reflection_r); printf("corneal reflection: (%d, %d)\n", corneal_reflection.x, corneal_reflection.y); - Draw_Cross(ellipse_image, corneal_reflection.x, corneal_reflection.y, 15, 15, Yellow); + Draw_Cross(ellipse_image, corneal_reflection.x, corneal_reflection.y, 15, 15, Yellow); //starburst pupil contour detection starburst_pupil_contour_detection((UINT8*)eye_image->imageData, eye_image->width, eye_image->height, edge_threshold, rays, min_feature_candidates); - + inliers_num = 0; inliers_index = pupil_fitting_inliers((UINT8*)eye_image->imageData, eye_image->width, eye_image->height, inliers_num); ellipse_axis.width = (int)pupil_param[0]; @@ -1022,8 +1554,8 @@ void process_image() Draw_Cross(ellipse_image, pupil.x, pupil.y, 15, 15, Red); cvLine(eye_image, pupil, corneal_reflection, Red, 4, 8); cvLine(ellipse_image, pupil, corneal_reflection, Red, 4, 8); - - printf("ellipse a:%lf; b:%lf, cx:%lf, cy:%lf, theta:%lf; inliers_num:%d\n\n", + + printf("ellipse a:%lf; b:%lf, cx:%lf, cy:%lf, theta:%lf; inliers_num:%d\n\n", pupil_param[0], pupil_param[1], pupil_param[2], pupil_param[3], pupil_param[4], inliers_num); bool is_inliers = 0; @@ -1053,10 +1585,10 @@ void process_image() diff_vector.y = pupil.y - corneal_reflection.y; if (do_map2scene) { gaze_point = homography_map_point(diff_vector); - printf("gaze_point: (%d,%d)\n", gaze_point.x, gaze_point.y); + printf("gaze_point: (%d,%d)\n", gaze_point.x, gaze_point.y); Draw_Cross(scene_image, gaze_point.x, gaze_point.y, 60, 60, Red); } - lost_frame_num = 0; + lost_frame_num = 0; } else { lost_frame_num++; } @@ -1075,42 +1607,44 @@ void process_image() fprintf(ellipse_log, "%.3f\t %8.2lf %8.2lf %8.2lf %8.2lf %8.2lf\n", Time_Elapsed(), pupil_param[0], pupil_param[1], pupil_param[2], pupil_param[3], pupil_param[4]); } - - printf("Time elapsed: %.3f\n", Time_Elapsed()); + + printf("Time elapsed: %.3f\n", Time_Elapsed()); fprintf(logfile,"%.3f\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n", Time_Elapsed(), - pupil.x, - pupil.y, - corneal_reflection.x, + pupil.x, + pupil.y, + corneal_reflection.x, corneal_reflection.y, - diff_vector.x, - diff_vector.y, - gaze_point.x, - gaze_point.y); + diff_vector.x, + diff_vector.y, + gaze_point.x, + gaze_point.y); if (view_cal_points) Show_Calibration_Points(); } -void Update_Gui_Windows() +void Update_Gui_Windows() { - //static int first = 1; + static int first = 1; cvShowImage(eye_window, eye_image); cvShowImage(original_eye_window, original_eye_image); cvReleaseImage(&original_eye_image); cvShowImage(scene_window, scene_image); cvShowImage(ellipse_window, ellipse_image); - cvResizeWindow(eye_window,320,240); - cvResizeWindow(original_eye_window,320,240); + //cvResizeWindow(eye_window,320,240); + //cvResizeWindow(original_eye_window,320,240); //cvResizeWindow(scene_window,320,240); cvResizeWindow(ellipse_window,320,240); - // only OpenCV 0.9.6 has the function of cvMoveWindow(), now we are using version 0.9.5 - /*if (first) { - cvMoveWindow(eye_window, 200, 0); - cvMoveWindow(scene_window, 200+320, 0); - cvMoveWindow(ellipse_window, 200, 240); - first = 0; - }*/ + // only OpenCV 0.9.6 has the function of cvMoveWindow(), + // does not work in version 0.9.5 + if (first) + { cvMoveWindow(original_eye_window,325, 0); + cvMoveWindow(eye_window, 325+320, 0); + cvMoveWindow(scene_window, 325, 270); + cvMoveWindow(ellipse_window, 325+320, 270); + first = 0; + } cvSetTrackbarPos("Edge Threshold", control_window, pupil_edge_thres); } @@ -1119,9 +1653,13 @@ void Open_GUI() { int i; + //Make the RGB eye image: + rgb_eye_image=cvCreateImageHeader(cvSize(640,480), IPL_DEPTH_8U, 3 ); /* VGA, 8 bits per color sample, 3 color channels */ + rgb_eye_image->imageData=(char *)malloc(640*480*3); /* 3 Bytes per pixel equals 921600 Bytes in RGB frame*/ + //Make the eye image (in monochrome): - eye_image=cvCreateImageHeader(cvSize(640,480), 8, 1 ); - eye_image->imageData=(char *)malloc(640*480); + eye_image=cvCreateImageHeader(cvSize(640,480), IPL_DEPTH_8U /* or just 8 */, 1 ); /* VGA, 8 bits per sample, 1 channel */ + eye_image->imageData=(char *)malloc(640*480); /* 1 Byte per pixel equals 307200 Bytes per frame */ //Make the eye image (in monochrome): threshold_image = cvCloneImage(eye_image); @@ -1129,8 +1667,8 @@ void Open_GUI() //Make the ellipse image (in RGB) : ellipse_image=cvCreateImageHeader(cvSize(640,480), 8, 3 ); ellipse_image->imageData=(char *)malloc(640*480*3); - - //Make the scene image: + + //Make the scene image: scene_image=cvCreateImageHeader(cvSize(640,480), 8, 3 ); scene_image->imageData=(char *)malloc(640*480*3); @@ -1141,9 +1679,11 @@ void Open_GUI() cvNamedWindow(eye_window, 0); cvNamedWindow(original_eye_window, 0); - //setup the mouse call back funtion here for calibration - cvSetMouseCallback(scene_window, on_mouse_scene); - cvSetMouseCallback(eye_window, on_mouse_eye); + /* PROBLEM */ + // setup the mouse call back function here for calibration + + cvSetMouseCallback(scene_window, on_mouse_scene); + cvSetMouseCallback(eye_window, on_mouse_eye); cvCreateTrackbar("Edge Threshold", control_window, &pupil_edge_thres, 255, NULL ); cvCreateTrackbar("Rays Number", control_window, &rays, 180, NULL ); @@ -1158,7 +1698,7 @@ void Open_GUI() Yellow = CV_RGB(255,255,0); } -void Close_GUI() +void Close_GUI() { cvDestroyWindow(eye_window); cvDestroyWindow(original_eye_window); @@ -1167,8 +1707,8 @@ void Close_GUI() cvDestroyWindow(control_window); cvReleaseImageHeader(&eye_image ); - cvReleaseImageHeader(&threshold_image ); - cvReleaseImageHeader(&original_eye_image ); + cvReleaseImageHeader(&threshold_image ); + cvReleaseImageHeader(&original_eye_image ); cvReleaseImageHeader(&ellipse_image ); cvReleaseImageHeader(&scene_image ); @@ -1179,11 +1719,11 @@ void Close_GUI() cvReleaseImage(&scene_image); } -void Open_Logfile(int argc, char** argv) +void Open_Logfile(int argc, char** argv) { char defaultlogfilename[]="logfile.txt"; char *logfilename; - + if (argc>1) { logfilename=argv[1]; } else { @@ -1200,14 +1740,14 @@ void Open_Logfile(int argc, char** argv) } } -void Close_Logfile() +void Close_Logfile() { fclose(logfile); } void Open_Ellipse_Log() { - static char *ellipse_log_name = "./Ellipse/ellipse_log.txt"; + char *ellipse_log_name = "./Ellipse/ellipse_log.txt"; ellipse_log = fopen(ellipse_log_name,"w+"); if (logfile!=NULL) { @@ -1222,7 +1762,31 @@ int main( int argc, char** argv ) { char c; - Open_IEEE1394(); +/* for firewire cameras */ +// Open_IEEE1394(); + +/* for v4l cameras */ + open_device (&fd, dev_name); + int dev_standard; + int dev_input; + int set_inp = 0; + int set_std = 0; + + //set the input if needed + // if (set_inp==1) + // set_input(&fd, dev_input); + + //set the standard if needed + // if (set_std==1) + // set_standard(&fd, dev_standard); + + buffers = init_device (&fd, dev_name, width, height, &n_buffers, 1); + //start_capturing (&fd, &n_buffers); + +/* ADD code for additional camera here */ +/* i'm not sure of this part: */ +// open_device (&fd, dev_name2); +// buffers = init_device (&fd, dev_name2, width, height, &n_buffers, 1); Open_GUI(); @@ -1253,10 +1817,8 @@ int main( int argc, char** argv ) } printf("\n"); } - - - + while ((c=cvWaitKey(50))!='q') { if (c == 's') { sprintf(eye_file, "eye%05d.bmp", image_no); @@ -1279,16 +1841,18 @@ int main( int argc, char** argv ) } if (start_point.x == -1 && start_point.y == -1) Grab_Camera_Frames(); - else - process_image(); - if (frame_number%1==0) Update_Gui_Windows(); + else + process_image(); + if (frame_number%1==0) Update_Gui_Windows(); } - Close_Logfile(); - - Close_GUI(); +Close_Logfile(); +Close_GUI(); - Close_IEEE1394(); +/* for firewire cameras */ +// Close_IEEE1394(); +/* for v4l cameras */ +Close_v4ldevices(); return 0; } From cd8bae4e1e83f263b0e7a73e6b2b9dfb144030c9 Mon Sep 17 00:00:00 2001 From: mareksuma1985 Date: Sat, 11 Jun 2016 21:04:17 +0200 Subject: [PATCH 2/4] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3900160..8e41e87 100755 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ LIBDIRS=-L/usr/local/lib -L/usr/X11R6/lib LIBS=-lm -ldc1394_control -lraw1394 -lopencv -lcvaux -lhighgui -INCLUDES=-I/usr/local/include/opencv +INCLUDES=-I/usr/include/opencv -I/usr/include/dc1394 DEFINES= -O2 From c4bf886057ecd6c1b32792716e740f9ab81a0d39 Mon Sep 17 00:00:00 2001 From: mareksuma1985 Date: Mon, 17 Dec 2018 01:31:30 +0100 Subject: [PATCH 3/4] fixed compilation error cvEyeTracker.c:425:18 --- cvEyeTracker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvEyeTracker.c b/cvEyeTracker.c index ef96af9..c31e4b0 100644 --- a/cvEyeTracker.c +++ b/cvEyeTracker.c @@ -422,7 +422,7 @@ static void start_capturing (int * fd, int * n_buffers ) static struct buffer *init_read (unsigned int buffer_size) { struct buffer *buffers = NULL; - buffers = calloc (1, sizeof (*buffers)); + buffers = (buffer*) calloc (1, sizeof (*buffers)); if (!buffers) { From 0c58ddb4fbc1e6de87bf7e59a8196781d116e79b Mon Sep 17 00:00:00 2001 From: mareksuma1985 Date: Mon, 17 Dec 2018 01:44:48 +0100 Subject: [PATCH 4/4] fixed compilation error ransac_ellipse.cpp:245:5 --- ransac_ellipse.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ransac_ellipse.cpp b/ransac_ellipse.cpp index 04e0798..c6788dc 100644 --- a/ransac_ellipse.cpp +++ b/ransac_ellipse.cpp @@ -27,9 +27,9 @@ * */ - #include #include +#include #include #include "ransac_ellipse.h" #include "svd.h"