PatchBatchExtractor.cpp

00001 /**************************************************
00002 
00003   PatchBatchExtractor class
00004 
00005   Antoni Chan,
00006   City University of Hong Kong, Copyright 2014
00007   libdt - OpenCV library for Dynamic Textures - version 1.0
00008 
00009   
00010 
00011 ***************************************************/
00012 
00013 #include <iostream>
00014 #include <string>
00015 #include<fstream>
00016 
00017 #include "opencv/cv.h"
00018 #include "opencv/cxcore.h"
00019 #include "opencv/highgui.h"
00020 
00021 
00022 #include "options.h"
00023 #include "utils/libutils.h"
00024 #include "video/PatchOptions.hpp"
00025 #include "video/PatchExtractor.hpp"
00026 #include "video/PatchBatchExtractor.hpp"
00027 #include "video/MatVid.hpp"
00028 
00029 //#define DEBUG
00030 
00031 /*
00032   Batch patch extraction
00033 */
00034 
00035 // constructor
00036 PatchBatchExtractor::PatchBatchExtractor(const Mat &vid, const PatchOptions & patopt,
00037                                          const Range &box_z, const Range &box_y, const Range &box_x) : 
00038   patopt(patopt)
00039 {
00040   CV_Assert(vid.dims == 3);    
00041 
00042   // remember settings
00043   this->box_x = box_x; this->box_y = box_y; this->box_z = box_z;
00044   this->vidsize = Point3i(vid.size[2], vid.size[1], vid.size[0]);
00045 
00046   // internal bounding box size and offsets
00047   Point3i vbox_size, vbox_off;
00048   // subvideo from internal bounding box
00049   Mat     boxvid;
00050 
00051   // check bounding box: 
00052   if ((box_z == Range::all()) && (box_y == Range::all()) && (box_z == Range::all())) {
00053     // using full video
00054     vbox_size = Point3i(vid.size[2], vid.size[1], vid.size[0]);
00055     vbox_off  = Point3i(0, 0, 0);
00056 
00057     boxvid = vid;
00058   } else {
00059     // using sub-video
00060     Range   vbox_z, vbox_x, vbox_y;
00061     // adjust box to align with patch locations
00062     if (box_z == Range::all()) {
00063       vbox_off.z  = 0;
00064       vbox_size.z = vid.size[0];
00065       vbox_z      = Range::all();
00066     } else {
00067       vbox_off.z  = (box_z.start % patopt.step.z == 0 ? box_z.start : ((box_z.start/patopt.step.z)+1)*patopt.step.z);
00068       vbox_size.z = box_z.end - vbox_off.z;
00069       vbox_z      = Range(vbox_off.z, MIN(box_z.end+patopt.win.z, vid.size[0]) );
00070     }
00071 
00072     if (box_y == Range::all()) {
00073       vbox_off.y  = 0;
00074       vbox_size.y = vid.size[1];
00075       vbox_y      = Range::all();
00076     } else {
00077       vbox_off.y  = (box_y.start % patopt.step.y == 0 ? box_y.start : ((box_y.start/patopt.step.y)+1)*patopt.step.y);
00078       vbox_size.y = box_y.end - vbox_off.y;
00079       vbox_y      = Range(vbox_off.y, MIN(box_y.end+patopt.win.y, vid.size[1]) );
00080     }
00081        
00082     if (box_x == Range::all()) {
00083       vbox_off.x  = 0;
00084       vbox_size.x = vid.size[2];
00085       vbox_x      = Range::all();
00086     } else {
00087       vbox_off.x  = (box_x.start % patopt.step.x == 0 ? box_x.start : ((box_x.start/patopt.step.x)+1)*patopt.step.x);
00088       vbox_size.x = box_x.end - vbox_off.x;
00089       vbox_x      = Range(vbox_off.x, MIN(box_x.end+patopt.win.x, vid.size[2]) );
00090     }    
00091 
00092     boxvid = MatVid::subvid(vid, vbox_z, vbox_y, vbox_x);
00093     
00094 #ifdef DEBUG
00095     cout << "box range.z = " << vbox_z << "\n";
00096     cout << "box range.y = " << vbox_y << "\n";
00097     cout << "box range.x = " << vbox_x << "\n";
00098 #endif
00099   }
00100 
00101 #ifdef DEBUG
00102   cout << "box size: " << vbox_size << "\n";
00103   cout << "box off:  " << vbox_off << "\n";
00104 #endif
00105 
00106   // create the patch extractor
00107   PatchExtractor pe(patopt, boxvid.size[1], boxvid.size[2]);
00108   
00109   // copy some info
00110   coff = pe.coff;
00111   allx = pe.allx;
00112   ally = pe.ally;
00113   int xsize = allx.size();
00114   int ysize = ally.size();
00115 
00116   // figure out possible z
00117   int zsize = (boxvid.size[0]-patopt.win.z) / patopt.step.z + 1;
00118   allz.reserve(zsize);
00119   for (int z=0; z<=boxvid.size[0]-patopt.win.z; z+=patopt.step.z) {
00120     allz.push_back(z);
00121   }
00122 
00123   // offset allx, ally, allz
00124   for (unsigned int i=0; i<allx.size(); i++)
00125     allx[i] += vbox_off.x;
00126   for (unsigned int i=0; i<ally.size(); i++)
00127     ally[i] += vbox_off.y;
00128   for (unsigned int i=0; i<allz.size(); i++)
00129     allz[i] += vbox_off.z;
00130 
00131   // reserve space
00132   int total = zsize*xsize*ysize;
00133   loc.reserve(total);
00134   patches.reserve(total);
00135   patchesall.reserve(total);
00136 #ifdef DEBUG
00137   cout << "reserve total: " << total << "\n";
00138 #endif
00139 
00140   // loop through frames
00141   int frame=0;
00142   Mat clip;
00143   while (frame < boxvid.size[0]) {
00144     // get next clip to add
00145     int clipz = (frame == 0 ? patopt.win.z : patopt.step.z);
00146 
00147 #ifdef DEBUG
00148     cout << "  @f=" << frame << ": read chunk of " << clipz << " frames\n";
00149 #endif
00150 
00151     if (frame+clipz > boxvid.size[0]) {
00152 
00153 #ifdef DEBUG
00154       cout << "  not enough frames\n";
00155 #endif
00156       break;
00157     }
00158     clip = MatVid::subvid(boxvid, Range(frame, frame+clipz), Range::all(), Range::all());
00159 
00160 #ifdef DEBUG
00161     dumpMatSize(clip);
00162 #endif
00163     // add frames
00164     if (!pe.addFrames(clip)) {
00165       // we should not get here
00166       CV_Error(-1, "something wrong!");
00167 
00168     } else {
00169       // now add to our patch vector
00170       const vector<Mat> &mypat = pe.getPatches();
00171       for (unsigned int i=0; i<mypat.size(); i++) 
00172           {
00173                 // create a copy of the patch
00174                   if(pe.locyx_mask[i]==true) //only valid patches
00175                   {
00176                         Mat p;
00177                         mypat[i].copyTo(p);
00178                         patches.push_back(p);
00179                         // add the location (with possible offset)
00180                         loc.push_back(Point3i(pe.locyx[i].x + vbox_off.x, 
00181                                                   pe.locyx[i].y + vbox_off.y, 
00182                                                   pe.locz       + vbox_off.z));
00183                         
00184                   }
00185                   //save all locations
00186                   locall.push_back(Point3i(pe.locyx[i].x + vbox_off.x, 
00187                           pe.locyx[i].y + vbox_off.y, 
00188                           pe.locz       + vbox_off.z));
00189                   locall_mask.push_back(pe.locyx_mask[i]);
00190                   
00191                   //save all patches
00192                   Mat p;
00193                   mypat[i].copyTo(p);
00194                   patchesall.push_back(p);
00195       }
00196     }
00197 
00198     frame += clipz;
00199   }
00200 }
00201 
00202 // desctructor
00203 PatchBatchExtractor::~PatchBatchExtractor() {
00204 }
00205 
00206 
00207 // debugging
00208 void PatchBatchExtractor::dump(ostream &output) const {
00209   output << "PatchBatchExtractor (" << patopt << "; ";
00210   output << "numpatches=" << patches.size() << "; ";
00211   output << "boxz=" << box_z << "; ";
00212   output << "boxy=" << box_y << "; ";
00213   output << "boxx=" << box_x << "; ";
00214   output << "vsize=" << vidsize << "; ";
00215   output << ")";
00216 }
00217 
00218 
00219 void PatchBatchExtractor::dump_matlab(ostream &output) const {
00220   output << "clear patb;\n";
00221   output << "patb.winsize = " << patopt.win << ";\n";
00222   output << "patb.step = " << patopt.step << ";\n";
00223   output << "patb.coff = " << coff << ";\n";
00224   output << "patb.xrange = " << Mat(allx) << "'+1;\n";
00225   output << "patb.yrange = " << Mat(ally) << "'+1;\n"; 
00226   output << "patb.zrange = " << Mat(allz) << "'+1;\n";
00227   if ((box_x == Range::all()) && (box_y == Range::all()) && (box_z == Range::all())) {
00228     // no output
00229   } else {
00230     output << "patb.bbox{1} = [" << (box_z == Range::all() ? 1 : box_z.start+1)
00231            << "," << (box_z == Range::all() ? vidsize.z : box_z.end) << "];\n";
00232     output << "patb.bbox{2} = [" << (box_y == Range::all() ? 1 : box_y.start+1)
00233            << "," << (box_y == Range::all() ? vidsize.y : box_y.end) << "];\n";
00234     output << "patb.bbox{3} = [" << (box_x == Range::all() ? 1 : box_x.start+1)
00235            << "," << (box_x == Range::all() ? vidsize.x : box_x.end) << "];\n";
00236   }
00237   output << "patb.ipatch = {};\n";
00238   output << "patb.loc = [];\n";
00239 
00240   for (unsigned int i=0; i<patches.size(); i++) {
00241     output << "patb.ipatch{end+1} = ";
00242     MatVid::dump_matlab(patches[i], output);
00243     output << ";\n";
00244     
00245     output << "patb.loc(end+1,:) = [" << loc[i].y << ","
00246            << loc[i].x << "," << loc[i].z << "]+1;\n";
00247   }
00248 }
00249 
00250 // convert to string
00251 ostream& operator<<(ostream& output, const PatchBatchExtractor& pat) {
00252   pat.dump(output);
00253   return output;
00254 }
00255 
00256 
00257 
00258 Mat PatchBatchExtractor::segm_mask(std::vector<int> &oldclasses)
00259 {
00260         Mat smask;
00261         int filledges=0;
00262         //fill in implicit background class
00263         std::vector<Point3i> myloc; 
00264         std::vector<int> classes;
00265         
00266         myloc=locall;
00267         int index=0;
00268         for(int i=0;i<locall_mask.size();i++)
00269         {
00270                 if(locall_mask[i]==true)
00271                 {
00272                         classes.push_back(oldclasses[index]);
00273                         index++;
00274                 }
00275                 else
00276                 {
00277                         classes.push_back(0);
00278                 }
00279         }
00280 
00281         
00282         // get locations
00283 std::vector<Point3i> loc;
00284         for(int i=0;i<myloc.size();i++)
00285         {
00286                 loc.push_back(myloc[i]+coff);
00287         }
00288         Point3i winsize=patopt.win;
00289         Point3i step=patopt.step;
00290 
00291         //check for single z-layer, and force mask to be 2d.
00292         if(allz.size()==1)
00293         {
00294                 for(int i=0;i<loc.size();i++)
00295                         loc[i].z=0;
00296 
00297                 winsize.z=1;
00298                 step.z=1;
00299                 vidsize.z=1;
00300         }
00301 
00302         int numclasses=*max_element(classes.begin(),classes.end());
00303 
00304         
00305         smask=MatVid::create(vidsize.z,vidsize.y,vidsize.x,CV_8UC1);
00306         smask=Scalar(0);
00307         int zsize=step.z;
00308 
00309         Point3i rwin;
00310         rwin.x=winsize.x/2;
00311         rwin.y=winsize.y/2;
00312         rwin.z=zsize/2;
00313 
00314         Point2i box_y(-rwin.x,winsize.x-1-rwin.x);
00315         Point2i box_x(-rwin.y,winsize.y-1-rwin.y);
00316         Point2i box_z(-rwin.z,zsize-1-rwin.z);
00317 
00318         int cy=rwin.x+1;
00319         int cx=rwin.y+1;
00320 
00321         //the structuring element
00322         Mat se(winsize.x,winsize.y,CV_8UC1);
00323         se=Scalar(0);
00324 
00325         //the distance matrix
00326         Mat sed(winsize.x,winsize.y,OPT_MAT_TYPE);
00327         sed=Scalar(0);
00328 
00329         std::vector<double> ssc;
00330         for(int i=0;i<sed.rows;i++)
00331                 for(int j=0;j<sed.cols;j++)
00332                 {
00333                         sed.at<OPT_F_TYPE>(i,j)=sqrt(  pow((double)(i+1-cy),2) + pow((double)(j+1-cx),2) );
00334                         ssc.push_back(sed.at<OPT_F_TYPE>(i,j));
00335                 }
00336                 
00337         //growing schedule
00338         sort(ssc.begin(),ssc.end());
00339         vector<double>::iterator it=unique(ssc.begin(),ssc.end());
00340         ssc.resize( it - ssc.begin() );
00341 
00342         //offsets
00343         std::vector<int> locoffs;
00344         for(int i=0;i<loc.size();i++)
00345         {
00346                 int temp=loc[i].y-rwin.x+vidsize.y*(loc[i].x-rwin.y)+vidsize.y*vidsize.x*(loc[i].z-rwin.z);
00347                 locoffs.push_back(temp);
00348         }
00349 
00350         //classes
00351         std::vector<int> cs=classes;
00352         sort(cs.begin(),cs.end());
00353         vector<int>::iterator it2=unique(cs.begin(),cs.end());
00354         cs.resize( it2 - cs.begin() );
00355 
00356         for(int seri=0;seri<ssc.size();seri++)
00357         {
00358                 cout<<"segm_mask: nn fill "<<seri<<"/"<<ssc.size()<<endl;
00359                 //build the structure element
00360                         
00361                 for(int i=0;i<sed.rows;i++)
00362                         for(int j=0;j<sed.cols;j++)
00363                         {
00364                                 if(sed.at<OPT_F_TYPE>(i,j)==ssc[seri])
00365                                         se.at<uchar>(i,j)=1;
00366                                 else
00367                                         se.at<uchar>(i,j)=0;
00368                         }
00369 
00370                                 
00371                         Mat se3=MatVid::repeat(se,zsize);
00372                         int s1=se3.size[0];
00373                         int s2=se3.size[1];
00374                         int s3=se3.size[2];
00375                         std::vector<double> se3i;
00376                         for(int kk=0;kk<s1;kk++)
00377                                 for(int jj=0;jj<s3;jj++)
00378                                         for(int ii=0;ii<s2;ii++)
00379                                         {
00380                                                 if(se3.at<uchar>(kk,ii,jj)==1)
00381                                                 {
00382                                                         double temp=ii+1+jj*vidsize.y+kk*vidsize.y*vidsize.x;
00383                                                         se3i.push_back(temp);
00384                                                 }
00385                                         }
00386                         for(int i=0;i<classes.size();i++)
00387                         {
00388                                 int c=classes[i];
00389                                 int off=locoffs[i];
00390 
00391                                 //find zero elements in smask that are active in se3
00392                                 std::vector<int> tempi;
00393                                 for(int j=0;j<se3i.size();j++)
00394                                 {
00395                                         int temp=se3i[j]+off-1;
00396                                         int fNo=temp/(smask.size[1]*smask.size[2]);
00397                                         temp=temp-fNo*(smask.size[1]*smask.size[2]);
00398                                         int colNo=temp/smask.size[1];
00399                                         int rowNo=temp%smask.size[1];
00400                                         if(smask.at<uchar>(fNo,rowNo,colNo)==0)
00401                                                 tempi.push_back(j);
00402                                 }
00403                                         
00404                                 //if found any zero elements, 
00405                                 for(int j=0;j<tempi.size();j++)
00406                                 {
00407                                         int temp=se3i[tempi[j]]+off-1;
00408                                         int fNo=temp/(smask.size[1]*smask.size[2]);
00409                                         temp=temp-fNo*(smask.size[1]*smask.size[2]);
00410                                         int colNo1=temp/smask.size[1];
00411                                         int rowNo1=temp%smask.size[1];
00412                                         // fill location
00413                                         if( (c==1) || (c==2) )
00414                                                 c=c;
00415                                         smask.at<uchar>(fNo,rowNo1,colNo1)=c;                                                   
00416                                 }
00417 
00418                         }
00419                         
00420         }
00421                                 
00422         //%%% now fill in the borders if they are missing %%%   
00423         double minx=*min_element(allx.begin(),allx.end())+coff.y+box_x.x;
00424         double maxx=*max_element(allx.begin(),allx.end())+coff.y+box_x.y+2;
00425         double miny=*min_element(ally.begin(),ally.end())+coff.x+box_y.x;
00426         double maxy=*max_element(ally.begin(),ally.end())+coff.x+box_y.y+2;
00427 
00428         if(1<=minx)
00429         {
00430                 // fill in left edge
00431                 for(int i=0;i<smask.size[0];i++)
00432                         for(int j=0;j<smask.size[1];j++)
00433                                 for(int k=0;k<minx;k++)
00434                                         smask.at<uchar>(i,j,k)=smask.at<uchar>(i,j,minx);
00435 
00436         }
00437 
00438         if(maxx<=vidsize.x)
00439         {
00440                 //fill in right edge
00441                 for(int i=0;i<smask.size[0];i++)
00442                         for(int j=0;j<smask.size[1];j++)
00443                                 for(int k=maxx-1;k<vidsize.x;k++)
00444                                         smask.at<uchar>(i,j,k)=smask.at<uchar>(i,j,maxx-2);
00445 
00446         }
00447 
00448         
00449 
00450         if(1<=miny)
00451         {
00452                 // fill in top edge
00453                 for(int i=0;i<smask.size[0];i++)
00454                         for(int j=0;j<miny;j++)
00455                                 for(int k=0;k<smask.size[2];k++)
00456                                         smask.at<uchar>(i,j,k)=smask.at<uchar>(i,miny,k);
00457 
00458         }
00459 
00460         if(maxy<=vidsize.y)
00461         {
00462                 // fill in bottom edge
00463                 for(int i=0;i<smask.size[0];i++)
00464                         for(int j=maxy-1;j<vidsize.y;j++)
00465                                 for(int k=0;k<smask.size[2];k++)
00466                                         smask.at<uchar>(i,j,k)=smask.at<uchar>(i,maxy-2,k);
00467 
00468         }
00469 
00470                         if(vidsize.z>1)
00471                         {
00472                                 double minz=*min_element(allz.begin(),allz.end())+coff.z+box_z.x;
00473                                 double maxz=*max_element(allz.begin(),allz.end())+coff.z+box_z.y+2;
00474 
00475                                 if(1<=minz)
00476                                 {
00477                                         //fill in first frames
00478                                         for(int i=0;i<minz;i++)
00479                                                 for(int j=0;j<smask.size[1];j++)
00480                                                         for(int k=0;k<smask.size[2];k++)
00481                                                         smask.at<uchar>(i,j,k)=smask.at<uchar>(minz,j,k);
00482                                 }
00483 
00484                                 if(maxz<=vidsize.z)
00485                                 {
00486                                         //fill in last frames
00487                                         for(int i=maxz-1;i<vidsize.z;i++)
00488                                                 for(int j=0;j<smask.size[1];j++)
00489                                                         for(int k=0;k<smask.size[2];k++)
00490                                                         smask.at<uchar>(i,j,k)=smask.at<uchar>(maxz-2,j,k);
00491                                 }
00492                         }
00493 
00494 
00495 /*                      
00496         FILE *fid=fopen("d:/bad/dump_mask_openCV.bin","wb");
00497         for(int i=0;i<smask.size[0];i++)
00498                 for(int j=0;j<smask.size[2];j++)
00499                         for(int k=0;k<smask.size[1];k++)
00500                         {
00501                                 double temp=smask.at<uchar>(i,k,j);
00502                                 fwrite(&temp,8,1,fid);
00503                         }
00504                         fclose(fid);*/
00505 
00506                         if(smask.size[0]==1)
00507                         {
00508                                 Mat tempM;
00509                                 MatVid::frame(smask,0).copyTo(tempM);
00510                                 return tempM;
00511                         }
00512                         else
00513                         {
00514                                 return smask;
00515                         }
00516 }
00517 

Generated on 18 Dec 2014 for libdt-OpenCV by  doxygen 1.6.1