function do_3dmouthgt( the_dir ) if nargin < 1 error( [ mfilename ' needs one input argument.' ] ); end d = dir( fullfile( the_dir, 'seq*.mouthgt' ) ); % Reject file names containing "-interpolated-reprojected" VIDEO_FRAMERATE = 25; % To get some information about recordings: use the "seq_av163.m" script. CORPUS_PATH = '../..'; addpath( CORPUS_PATH ); % Loop through input files for i_file = 1:length( d ) considerit = ~d( i_file ).isdir & ~isempty( strfind( d( i_file ).name, 'cam1' ) ) & isempty( strfind( d( i_file ).name, 'interpolated' ) ) & isempty( strfind( d( i_file ).name, 'reprojected' ) ); if considerit IN_NAME1 = fullfile( the_dir, d( i_file ).name ); IN_NAME2 = strrep( IN_NAME1, 'cam1', 'cam2' ); IN_NAME3 = strrep( IN_NAME1, 'cam1', 'cam3' ); if exist( IN_NAME2, 'file' ) & exist( IN_NAME3, 'file' ) [ p,n,e ] = fileparts( d( i_file ).name ); ind = strfind( n, 'cam1' ); seqname = n( 1:ind(1)-2 ); n = strrep( n, '-cam1', '' ); OUT_NAME3D = fullfile( the_dir, [ p n '.3dmouthgt' ] ); % Files that will contain: % - original 2D manual measurements. % - between manual measurements: 2D reprojections of the 3D location estimates. % % The intent is to check visually whether or not the 3D reconstruction worked. [ p,n,e ] = fileparts( IN_NAME1 ); OUT_NAME1 = fullfile( p, [ n '-reprojected' e ] ); [ p,n,e ] = fileparts( IN_NAME2 ); OUT_NAME2 = fullfile( p, [ n '-reprojected' e ] ); [ p,n,e ] = fileparts( IN_NAME3 ); OUT_NAME3 = fullfile( p, [ n '-reprojected' e ] ); disp( ' ' ); disp( [ 'IN_NAME1: "' IN_NAME1 '"' ] ); disp( [ 'IN_NAME2: "' IN_NAME2 '"' ] ); disp( [ 'IN_NAME3: "' IN_NAME3 '"' ] ); disp( [ 'OUT_NAME3D: "' OUT_NAME3D '"' ] ); disp( [ 'OUT_NAME1: "' OUT_NAME1 '"' ] ); disp( [ 'OUT_NAME2: "' OUT_NAME2 '"' ] ); disp( [ 'OUT_NAME3: "' OUT_NAME3 '"' ] ); % Load for a_cam = 1:3 cam( a_cam ).mouthgt = load( eval( sprintf( 'IN_NAME%d', a_cam ) ), '-ASCII' ); end %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % ( 1 ) Load GT files and details (timecode) all_timestamps = []; o = seq_av163( seqname, CORPUS_PATH ); for a = 1:3 % Save it as is cam( a ).mouthgt_orig = cam( a ).mouthgt; % Convert frame # into time stamp cam( a ).mouthgt( :, 1 ) = o.cam( a ).frame1_timestamp + ( cam( a ).mouthgt( :, 1 ) - 1 ) / VIDEO_FRAMERATE; % Make sure the time stamp is unique cam( a ).mouthgt( :, 1 ) = round( cam( a ).mouthgt( :, 1 ) * 1e6 ) * 1e-6; % Keep only manual GT, % discard possible results of previous interpolation cam( a ).mouthgt = cam( a ).mouthgt( find( cam( a ).mouthgt( :, 2 ) == 1 ), : ); % Keep only information about a visible mouth! cam( a ).mouthgt = cam( a ).mouthgt( find( cam( a ).mouthgt( :, 3 ) == 1 ), : ); % Fix the minor pixel variations in (X,Y) origin between sessions cam( a ).mouthgt( :, 4 ) = cam( a ).mouthgt( :, 4 ) - o.calib.cam( a ).delta_x_from_session08; cam( a ).mouthgt( :, 5 ) = cam( a ).mouthgt( :, 5 ) - o.calib.cam( a ).delta_y_from_session08; % Stack up the list of all timestamps all_timestamps = [ all_timestamps; cam( a ).mouthgt( :, 1 ) ]; end all_timestamps = sort( unique( all_timestamps ) ); %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % ( 2 ) Reconstruct 3D mouth location when possible % Where we will store the result % - column 1: time stamp (in seconds) % - column 2: "is_manual" flag (0:interpolated or 1:manual) % - column 3 to 5: x,y,z coordinates mouthgt3d = zeros( 0, 5 ); % Prepare things for "reconstruct_3d" for a = 1:3 cam( a ).Pmat = o.calib.cam( a ).session08_Pmat; cam( a ).K = o.calib.cam( a ).session08_K; cam( a ).kc = o.calib.cam( a ).session08_kc; cam( a ).alpha_c = o.calib.cam( a ).session08_alpha_c; end % Process data for t = all_timestamps(:).' % For each timestamp, load corresponding 2D measurements ind_cam = [ NaN NaN NaN ]; p2d = repmat( NaN, 3, 3 ); for a = 1:3 ind = find( cam( a ).mouthgt == t ); if length( ind ) == 1 ind_cam( a ) = ind; p2d( :, a ) = [ cam( a ).mouthgt( ind_cam( a ), 4:5 ).'; 1 ]; end end % binary vector : 0 for unavailable measurement, 1 for available measurement avail = ~isnan( ind_cam ); which_avail = find( avail ); p3d = []; if all( avail ) | all( avail == [ 1 1 0 ] ) | all( avail == [ 1 0 1 ] ) % cams [ 1,2,3 ] or cams [ 1,2 ] or cams [ 1,3 ] % NOT cams [ 2,3 ] because not precise enough in my opinion align_Pmat = eval( [ 'o.calib.rigid' sprintf( '%02d', which_avail ) '_Pmat' ] ); p3d = reconstruct_3d( reshape( p2d( :, which_avail ), [], 1 ), cam( which_avail ), align_Pmat ); % Store the result % the "1" means that this 3D location estimate % was derived from manual measurements mouthgt3d( end+1, : ) = [ t 1 p3d( 1:3 ).' ]; % Sort the matrix by increasing time stamp value [tmp, ind] = sort( mouthgt3d( :, 1 ) ); mouthgt3d = mouthgt3d( ind, : ); end end % Save the 3D ground-truth save_3dmouthgt( OUT_NAME3D, mouthgt3d ); %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % ( 3 ) Reprojection on the image plane of each camera m = mouthgt3d( :, 3:5 ).'; m( 4,: ) = 1; p2d_reproj = project( m, cam, o.calib.rigid010203_Pmat ); % Put the small pixel deviation (rel. to session08) back in for a = 1:3 p2d_reproj( 1 + (a-1)*3, : ) = p2d_reproj( 1 + (a-1)*3, : ) + o.calib.cam( a ).delta_x_from_session08; p2d_reproj( 2 + (a-1)*3, : ) = p2d_reproj( 2 + (a-1)*3, : ) + o.calib.cam( a ).delta_y_from_session08; end % Insert the new points in the GT of each camera % Save it to a file t_reproj = mouthgt3d( :, 1 ); for a = 1:3 % Translate manual gt back from "timestamp (in seconds)" to "frame number (integer)" cam( a ).mouthgt( :, 1 ) = 1 + round( ( cam( a ).mouthgt( :, 1 ) - o.cam( a ).frame1_timestamp ) * VIDEO_FRAMERATE ); % Add the new reprojected 2D measuremenets % (drop old ones in case of conflict) frame_reproj = 1 + round( ( t_reproj - o.cam( a ).frame1_timestamp ) * VIDEO_FRAMERATE ); n = length( frame_reproj ); new_mouthgt = [ frame_reproj(:), repmat( [ 0 1 ], n, 1 ), p2d_reproj( (1:2) + (a-1)*3, : ).' ]; ind = find( ~ismember( cam( a ).mouthgt_orig( :, 1 ), new_mouthgt( :,1 ) ) ); cam( a ).mouthgt = [ cam( a ).mouthgt_orig( ind,: ); new_mouthgt ]; % Sort by increasing time index [ tmp, ind ] = sort( cam( a ).mouthgt( :,1 ) ); cam( a ).mouthgt = cam( a ).mouthgt( ind, : ); % Write out output2d_file = eval( sprintf( 'OUT_NAME%d', a ) ); save_mouthgt( output2d_file, cam( a ).mouthgt ); % Compute 2D reprojection error % is_manual & is_visible ind = find( all( cam( a ).mouthgt_orig( :, 2:3 ), 2 ) ); f = intersect( cam( a ).mouthgt( :,1 ), cam( a ).mouthgt_orig( ind,1 ) ); ind_result = find( ismember( cam( a ).mouthgt( :,1 ), f ) ); ind_orig = find( ismember( cam( a ).mouthgt_orig( :,1 ), f ) ); delta = cam( a ).mouthgt( ind_result, 4:5 ) - cam( a ).mouthgt_orig( ind_orig, 4:5 ); ind = find( delta(:, 1 ) & delta( :,2 ) ); repr_error = sqrt( mean( delta( ind,:) .^ 2 ) ); disp( sprintf( 'cam%d: 3D->2D reprojection error: [ dx:%6.3f, dy:%6.3f ] pixels', a, repr_error ) ); [ tmp, ind_max ] = max( abs( delta( ind, 1 ) ) ); f_dx_max = cam( a ).mouthgt( ind_result( ind( ind_max ) ), 1 ); [ tmp, ind_max ] = max( abs( delta( ind, 2 ) ) ); f_dy_max = cam( a ).mouthgt( ind_result( ind( ind_max ) ), 1 ); disp( sprintf( ' max dx error on frame #%d, max dy error on frame #%d', f_dx_max, f_dy_max ) ); end end end end