00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013 #include <iostream>
00014 #include <string>
00015 #include<fstream>
00016
00017 #include "opencv/cv.h"
00018 #include "opencv/cxcore.h"
00019 #include "opencv/highgui.h"
00020
00021
00022 #include "options.h"
00023 #include "utils/libutils.h"
00024 #include "video/PatchOptions.hpp"
00025 #include "video/PatchExtractor.hpp"
00026 #include "video/PatchBatchExtractor.hpp"
00027 #include "video/MatVid.hpp"
00028
00029
00030
00031
00032
00033
00034
00035
00036 PatchBatchExtractor::PatchBatchExtractor(const Mat &vid, const PatchOptions & patopt,
00037 const Range &box_z, const Range &box_y, const Range &box_x) :
00038 patopt(patopt)
00039 {
00040 CV_Assert(vid.dims == 3);
00041
00042
00043 this->box_x = box_x; this->box_y = box_y; this->box_z = box_z;
00044 this->vidsize = Point3i(vid.size[2], vid.size[1], vid.size[0]);
00045
00046
00047 Point3i vbox_size, vbox_off;
00048
00049 Mat boxvid;
00050
00051
00052 if ((box_z == Range::all()) && (box_y == Range::all()) && (box_z == Range::all())) {
00053
00054 vbox_size = Point3i(vid.size[2], vid.size[1], vid.size[0]);
00055 vbox_off = Point3i(0, 0, 0);
00056
00057 boxvid = vid;
00058 } else {
00059
00060 Range vbox_z, vbox_x, vbox_y;
00061
00062 if (box_z == Range::all()) {
00063 vbox_off.z = 0;
00064 vbox_size.z = vid.size[0];
00065 vbox_z = Range::all();
00066 } else {
00067 vbox_off.z = (box_z.start % patopt.step.z == 0 ? box_z.start : ((box_z.start/patopt.step.z)+1)*patopt.step.z);
00068 vbox_size.z = box_z.end - vbox_off.z;
00069 vbox_z = Range(vbox_off.z, MIN(box_z.end+patopt.win.z, vid.size[0]) );
00070 }
00071
00072 if (box_y == Range::all()) {
00073 vbox_off.y = 0;
00074 vbox_size.y = vid.size[1];
00075 vbox_y = Range::all();
00076 } else {
00077 vbox_off.y = (box_y.start % patopt.step.y == 0 ? box_y.start : ((box_y.start/patopt.step.y)+1)*patopt.step.y);
00078 vbox_size.y = box_y.end - vbox_off.y;
00079 vbox_y = Range(vbox_off.y, MIN(box_y.end+patopt.win.y, vid.size[1]) );
00080 }
00081
00082 if (box_x == Range::all()) {
00083 vbox_off.x = 0;
00084 vbox_size.x = vid.size[2];
00085 vbox_x = Range::all();
00086 } else {
00087 vbox_off.x = (box_x.start % patopt.step.x == 0 ? box_x.start : ((box_x.start/patopt.step.x)+1)*patopt.step.x);
00088 vbox_size.x = box_x.end - vbox_off.x;
00089 vbox_x = Range(vbox_off.x, MIN(box_x.end+patopt.win.x, vid.size[2]) );
00090 }
00091
00092 boxvid = MatVid::subvid(vid, vbox_z, vbox_y, vbox_x);
00093
00094 #ifdef DEBUG
00095 cout << "box range.z = " << vbox_z << "\n";
00096 cout << "box range.y = " << vbox_y << "\n";
00097 cout << "box range.x = " << vbox_x << "\n";
00098 #endif
00099 }
00100
00101 #ifdef DEBUG
00102 cout << "box size: " << vbox_size << "\n";
00103 cout << "box off: " << vbox_off << "\n";
00104 #endif
00105
00106
00107 PatchExtractor pe(patopt, boxvid.size[1], boxvid.size[2]);
00108
00109
00110 coff = pe.coff;
00111 allx = pe.allx;
00112 ally = pe.ally;
00113 int xsize = allx.size();
00114 int ysize = ally.size();
00115
00116
00117 int zsize = (boxvid.size[0]-patopt.win.z) / patopt.step.z + 1;
00118 allz.reserve(zsize);
00119 for (int z=0; z<=boxvid.size[0]-patopt.win.z; z+=patopt.step.z) {
00120 allz.push_back(z);
00121 }
00122
00123
00124 for (unsigned int i=0; i<allx.size(); i++)
00125 allx[i] += vbox_off.x;
00126 for (unsigned int i=0; i<ally.size(); i++)
00127 ally[i] += vbox_off.y;
00128 for (unsigned int i=0; i<allz.size(); i++)
00129 allz[i] += vbox_off.z;
00130
00131
00132 int total = zsize*xsize*ysize;
00133 loc.reserve(total);
00134 patches.reserve(total);
00135 patchesall.reserve(total);
00136 #ifdef DEBUG
00137 cout << "reserve total: " << total << "\n";
00138 #endif
00139
00140
00141 int frame=0;
00142 Mat clip;
00143 while (frame < boxvid.size[0]) {
00144
00145 int clipz = (frame == 0 ? patopt.win.z : patopt.step.z);
00146
00147 #ifdef DEBUG
00148 cout << " @f=" << frame << ": read chunk of " << clipz << " frames\n";
00149 #endif
00150
00151 if (frame+clipz > boxvid.size[0]) {
00152
00153 #ifdef DEBUG
00154 cout << " not enough frames\n";
00155 #endif
00156 break;
00157 }
00158 clip = MatVid::subvid(boxvid, Range(frame, frame+clipz), Range::all(), Range::all());
00159
00160 #ifdef DEBUG
00161 dumpMatSize(clip);
00162 #endif
00163
00164 if (!pe.addFrames(clip)) {
00165
00166 CV_Error(-1, "something wrong!");
00167
00168 } else {
00169
00170 const vector<Mat> &mypat = pe.getPatches();
00171 for (unsigned int i=0; i<mypat.size(); i++)
00172 {
00173
00174 if(pe.locyx_mask[i]==true)
00175 {
00176 Mat p;
00177 mypat[i].copyTo(p);
00178 patches.push_back(p);
00179
00180 loc.push_back(Point3i(pe.locyx[i].x + vbox_off.x,
00181 pe.locyx[i].y + vbox_off.y,
00182 pe.locz + vbox_off.z));
00183
00184 }
00185
00186 locall.push_back(Point3i(pe.locyx[i].x + vbox_off.x,
00187 pe.locyx[i].y + vbox_off.y,
00188 pe.locz + vbox_off.z));
00189 locall_mask.push_back(pe.locyx_mask[i]);
00190
00191
00192 Mat p;
00193 mypat[i].copyTo(p);
00194 patchesall.push_back(p);
00195 }
00196 }
00197
00198 frame += clipz;
00199 }
00200 }
00201
00202
00203 PatchBatchExtractor::~PatchBatchExtractor() {
00204 }
00205
00206
00207
00208 void PatchBatchExtractor::dump(ostream &output) const {
00209 output << "PatchBatchExtractor (" << patopt << "; ";
00210 output << "numpatches=" << patches.size() << "; ";
00211 output << "boxz=" << box_z << "; ";
00212 output << "boxy=" << box_y << "; ";
00213 output << "boxx=" << box_x << "; ";
00214 output << "vsize=" << vidsize << "; ";
00215 output << ")";
00216 }
00217
00218
00219 void PatchBatchExtractor::dump_matlab(ostream &output) const {
00220 output << "clear patb;\n";
00221 output << "patb.winsize = " << patopt.win << ";\n";
00222 output << "patb.step = " << patopt.step << ";\n";
00223 output << "patb.coff = " << coff << ";\n";
00224 output << "patb.xrange = " << Mat(allx) << "'+1;\n";
00225 output << "patb.yrange = " << Mat(ally) << "'+1;\n";
00226 output << "patb.zrange = " << Mat(allz) << "'+1;\n";
00227 if ((box_x == Range::all()) && (box_y == Range::all()) && (box_z == Range::all())) {
00228
00229 } else {
00230 output << "patb.bbox{1} = [" << (box_z == Range::all() ? 1 : box_z.start+1)
00231 << "," << (box_z == Range::all() ? vidsize.z : box_z.end) << "];\n";
00232 output << "patb.bbox{2} = [" << (box_y == Range::all() ? 1 : box_y.start+1)
00233 << "," << (box_y == Range::all() ? vidsize.y : box_y.end) << "];\n";
00234 output << "patb.bbox{3} = [" << (box_x == Range::all() ? 1 : box_x.start+1)
00235 << "," << (box_x == Range::all() ? vidsize.x : box_x.end) << "];\n";
00236 }
00237 output << "patb.ipatch = {};\n";
00238 output << "patb.loc = [];\n";
00239
00240 for (unsigned int i=0; i<patches.size(); i++) {
00241 output << "patb.ipatch{end+1} = ";
00242 MatVid::dump_matlab(patches[i], output);
00243 output << ";\n";
00244
00245 output << "patb.loc(end+1,:) = [" << loc[i].y << ","
00246 << loc[i].x << "," << loc[i].z << "]+1;\n";
00247 }
00248 }
00249
00250
00251 ostream& operator<<(ostream& output, const PatchBatchExtractor& pat) {
00252 pat.dump(output);
00253 return output;
00254 }
00255
00256
00257
00258 Mat PatchBatchExtractor::segm_mask(std::vector<int> &oldclasses)
00259 {
00260 Mat smask;
00261 int filledges=0;
00262
00263 std::vector<Point3i> myloc;
00264 std::vector<int> classes;
00265
00266 myloc=locall;
00267 int index=0;
00268 for(int i=0;i<locall_mask.size();i++)
00269 {
00270 if(locall_mask[i]==true)
00271 {
00272 classes.push_back(oldclasses[index]);
00273 index++;
00274 }
00275 else
00276 {
00277 classes.push_back(0);
00278 }
00279 }
00280
00281
00282
00283 std::vector<Point3i> loc;
00284 for(int i=0;i<myloc.size();i++)
00285 {
00286 loc.push_back(myloc[i]+coff);
00287 }
00288 Point3i winsize=patopt.win;
00289 Point3i step=patopt.step;
00290
00291
00292 if(allz.size()==1)
00293 {
00294 for(int i=0;i<loc.size();i++)
00295 loc[i].z=0;
00296
00297 winsize.z=1;
00298 step.z=1;
00299 vidsize.z=1;
00300 }
00301
00302 int numclasses=*max_element(classes.begin(),classes.end());
00303
00304
00305 smask=MatVid::create(vidsize.z,vidsize.y,vidsize.x,CV_8UC1);
00306 smask=Scalar(0);
00307 int zsize=step.z;
00308
00309 Point3i rwin;
00310 rwin.x=winsize.x/2;
00311 rwin.y=winsize.y/2;
00312 rwin.z=zsize/2;
00313
00314 Point2i box_y(-rwin.x,winsize.x-1-rwin.x);
00315 Point2i box_x(-rwin.y,winsize.y-1-rwin.y);
00316 Point2i box_z(-rwin.z,zsize-1-rwin.z);
00317
00318 int cy=rwin.x+1;
00319 int cx=rwin.y+1;
00320
00321
00322 Mat se(winsize.x,winsize.y,CV_8UC1);
00323 se=Scalar(0);
00324
00325
00326 Mat sed(winsize.x,winsize.y,OPT_MAT_TYPE);
00327 sed=Scalar(0);
00328
00329 std::vector<double> ssc;
00330 for(int i=0;i<sed.rows;i++)
00331 for(int j=0;j<sed.cols;j++)
00332 {
00333 sed.at<OPT_F_TYPE>(i,j)=sqrt( pow((double)(i+1-cy),2) + pow((double)(j+1-cx),2) );
00334 ssc.push_back(sed.at<OPT_F_TYPE>(i,j));
00335 }
00336
00337
00338 sort(ssc.begin(),ssc.end());
00339 vector<double>::iterator it=unique(ssc.begin(),ssc.end());
00340 ssc.resize( it - ssc.begin() );
00341
00342
00343 std::vector<int> locoffs;
00344 for(int i=0;i<loc.size();i++)
00345 {
00346 int temp=loc[i].y-rwin.x+vidsize.y*(loc[i].x-rwin.y)+vidsize.y*vidsize.x*(loc[i].z-rwin.z);
00347 locoffs.push_back(temp);
00348 }
00349
00350
00351 std::vector<int> cs=classes;
00352 sort(cs.begin(),cs.end());
00353 vector<int>::iterator it2=unique(cs.begin(),cs.end());
00354 cs.resize( it2 - cs.begin() );
00355
00356 for(int seri=0;seri<ssc.size();seri++)
00357 {
00358 cout<<"segm_mask: nn fill "<<seri<<"/"<<ssc.size()<<endl;
00359
00360
00361 for(int i=0;i<sed.rows;i++)
00362 for(int j=0;j<sed.cols;j++)
00363 {
00364 if(sed.at<OPT_F_TYPE>(i,j)==ssc[seri])
00365 se.at<uchar>(i,j)=1;
00366 else
00367 se.at<uchar>(i,j)=0;
00368 }
00369
00370
00371 Mat se3=MatVid::repeat(se,zsize);
00372 int s1=se3.size[0];
00373 int s2=se3.size[1];
00374 int s3=se3.size[2];
00375 std::vector<double> se3i;
00376 for(int kk=0;kk<s1;kk++)
00377 for(int jj=0;jj<s3;jj++)
00378 for(int ii=0;ii<s2;ii++)
00379 {
00380 if(se3.at<uchar>(kk,ii,jj)==1)
00381 {
00382 double temp=ii+1+jj*vidsize.y+kk*vidsize.y*vidsize.x;
00383 se3i.push_back(temp);
00384 }
00385 }
00386 for(int i=0;i<classes.size();i++)
00387 {
00388 int c=classes[i];
00389 int off=locoffs[i];
00390
00391
00392 std::vector<int> tempi;
00393 for(int j=0;j<se3i.size();j++)
00394 {
00395 int temp=se3i[j]+off-1;
00396 int fNo=temp/(smask.size[1]*smask.size[2]);
00397 temp=temp-fNo*(smask.size[1]*smask.size[2]);
00398 int colNo=temp/smask.size[1];
00399 int rowNo=temp%smask.size[1];
00400 if(smask.at<uchar>(fNo,rowNo,colNo)==0)
00401 tempi.push_back(j);
00402 }
00403
00404
00405 for(int j=0;j<tempi.size();j++)
00406 {
00407 int temp=se3i[tempi[j]]+off-1;
00408 int fNo=temp/(smask.size[1]*smask.size[2]);
00409 temp=temp-fNo*(smask.size[1]*smask.size[2]);
00410 int colNo1=temp/smask.size[1];
00411 int rowNo1=temp%smask.size[1];
00412
00413 if( (c==1) || (c==2) )
00414 c=c;
00415 smask.at<uchar>(fNo,rowNo1,colNo1)=c;
00416 }
00417
00418 }
00419
00420 }
00421
00422
00423 double minx=*min_element(allx.begin(),allx.end())+coff.y+box_x.x;
00424 double maxx=*max_element(allx.begin(),allx.end())+coff.y+box_x.y+2;
00425 double miny=*min_element(ally.begin(),ally.end())+coff.x+box_y.x;
00426 double maxy=*max_element(ally.begin(),ally.end())+coff.x+box_y.y+2;
00427
00428 if(1<=minx)
00429 {
00430
00431 for(int i=0;i<smask.size[0];i++)
00432 for(int j=0;j<smask.size[1];j++)
00433 for(int k=0;k<minx;k++)
00434 smask.at<uchar>(i,j,k)=smask.at<uchar>(i,j,minx);
00435
00436 }
00437
00438 if(maxx<=vidsize.x)
00439 {
00440
00441 for(int i=0;i<smask.size[0];i++)
00442 for(int j=0;j<smask.size[1];j++)
00443 for(int k=maxx-1;k<vidsize.x;k++)
00444 smask.at<uchar>(i,j,k)=smask.at<uchar>(i,j,maxx-2);
00445
00446 }
00447
00448
00449
00450 if(1<=miny)
00451 {
00452
00453 for(int i=0;i<smask.size[0];i++)
00454 for(int j=0;j<miny;j++)
00455 for(int k=0;k<smask.size[2];k++)
00456 smask.at<uchar>(i,j,k)=smask.at<uchar>(i,miny,k);
00457
00458 }
00459
00460 if(maxy<=vidsize.y)
00461 {
00462
00463 for(int i=0;i<smask.size[0];i++)
00464 for(int j=maxy-1;j<vidsize.y;j++)
00465 for(int k=0;k<smask.size[2];k++)
00466 smask.at<uchar>(i,j,k)=smask.at<uchar>(i,maxy-2,k);
00467
00468 }
00469
00470 if(vidsize.z>1)
00471 {
00472 double minz=*min_element(allz.begin(),allz.end())+coff.z+box_z.x;
00473 double maxz=*max_element(allz.begin(),allz.end())+coff.z+box_z.y+2;
00474
00475 if(1<=minz)
00476 {
00477
00478 for(int i=0;i<minz;i++)
00479 for(int j=0;j<smask.size[1];j++)
00480 for(int k=0;k<smask.size[2];k++)
00481 smask.at<uchar>(i,j,k)=smask.at<uchar>(minz,j,k);
00482 }
00483
00484 if(maxz<=vidsize.z)
00485 {
00486
00487 for(int i=maxz-1;i<vidsize.z;i++)
00488 for(int j=0;j<smask.size[1];j++)
00489 for(int k=0;k<smask.size[2];k++)
00490 smask.at<uchar>(i,j,k)=smask.at<uchar>(maxz-2,j,k);
00491 }
00492 }
00493
00494
00495
00496
00497
00498
00499
00500
00501
00502
00503
00504
00505
00506 if(smask.size[0]==1)
00507 {
00508 Mat tempM;
00509 MatVid::frame(smask,0).copyTo(tempM);
00510 return tempM;
00511 }
00512 else
00513 {
00514 return smask;
00515 }
00516 }
00517