PatchExtractor.cpp

00001 /**************************************************
00002 
00003   PatchExtractor class
00004 
00005   Antoni Chan,
00006   City University of Hong Kong, Copyright 2014
00007   libdt - OpenCV library for Dynamic Textures - version 1.0
00008 
00009   
00010 
00011 ***************************************************/
00012 
00013 #include <iostream>
00014 #include <string>
00015 #include<fstream>
00016 
00017 #include "opencv/cv.h"
00018 #include "opencv/cxcore.h"
00019 #include "opencv/highgui.h"
00020 
00021 #include "options.h"
00022 #include "utils/libutils.h"
00023 #include "video/PatchOptions.hpp"
00024 #include "video/PatchExtractor.hpp"
00025 #include "video/MatVid.hpp"
00026 
00027 //#define DEBUG
00028 
00029 /*
00030   there are several modes:
00031   1) norm_none --> patches are sub-videos of the buffer (vbuf)
00032   2) norm_zm   --> patches are sub-videos of the buffer (vbuf_zm)
00033   3) norm_zmuv --> patches are local copies
00034 */
00035 
00036 
00037 // constructor
00038 PatchExtractor::PatchExtractor(const PatchOptions & patopt, int vrows, int vcols) : 
00039   patopt(patopt), vrows(vrows), vcols(vcols)
00040 {
00041   CV_Assert((vrows>=patopt.win.y) && (vcols>=patopt.win.x));
00042 
00043   // initialize the location structure
00044   getLoc();
00045 
00046   // initialize the buffers
00047   vbuf = MatVid::create(patopt.win.z, vrows, vcols, OPT_MAT_TYPE);
00048   if (patopt.normopt != PatchOptions::NORM_NONE) {
00049     vbuf_zm = MatVid::create(patopt.win.z, vrows, vcols, OPT_MAT_TYPE);
00050     flag_zm = true;
00051   } else {
00052     flag_zm = false;
00053   }
00054 
00055   // initialize patches
00056   patches.resize(locyx.size());
00057   switch(patopt.normopt) {
00058   case PatchOptions::NORM_NONE:
00059     // setup patches on the vbuf buffer
00060     for (int i=0; i<(int)locyx.size(); i++) {
00061       patches[i] = MatVid::subvid(vbuf, Range(0, patopt.win.z), 
00062                                   Range(locyx[i].y, locyx[i].y+patopt.win.y), 
00063                                   Range(locyx[i].x, locyx[i].x+patopt.win.x));
00064     }
00065     break;
00066   case PatchOptions::NORM_ZM:
00067     // setup patches on the vbuf_zm buffer
00068     for (int i=0; i<(int)locyx.size(); i++) {
00069       patches[i] = MatVid::subvid(vbuf_zm, Range(0, patopt.win.z), 
00070                                   Range(locyx[i].y, locyx[i].y+patopt.win.y), 
00071                                   Range(locyx[i].x, locyx[i].x+patopt.win.x));
00072     }
00073     break;
00074   case PatchOptions::NORM_ZMUV:
00075     // setup patches (allocate new memory)
00076     for (int i=0; i<(int)locyx.size(); i++) {
00077       patches[i] = MatVid::create(patopt.win.z, patopt.win.y, patopt.win.x, OPT_MAT_TYPE);
00078     }
00079     break;
00080   default:
00081     CV_Error(-1, "bad option");
00082   }
00083   
00084   // reset the extractor
00085   reset();
00086   
00087   
00088   // initialize pointer to center frame
00089   frame_center = MatVid::frame(vbuf, coff.z);
00090 
00091   // initialize pointer to center chunk of frames
00092   // The number of frames will be the z-step size.
00093   // TODO: Except on the first call, which will also include the beginning frames.
00094   int tmp = patopt.step.z / 2;
00095   frame_patches = MatVid::subvid(vbuf, 
00096                                  Range(coff.z-tmp, coff.z-tmp+patopt.step.z),
00097                                  Range::all(), Range::all());
00098   //dumpMatSize(frame_patches);
00099 }
00100 
00101 // desctructor
00102 PatchExtractor::~PatchExtractor() {
00103 }
00104 
00105 // get locations
00106 void PatchExtractor::getLoc() {
00107   
00108 #ifdef DEBUG
00109   cout << "size: " << vrows << "," << vcols << "\n";
00110 #endif
00111 
00112   // coff = floor(winsize/2)
00113   
00114   //BUG FIXED
00115   //coff = (Point3i)(patopt.win * 0.5);
00116 
00117   coff.x=patopt.win.x*0.5;
00118   coff.y=patopt.win.y*0.5;
00119   coff.z=patopt.win.z*0.5;
00120 
00121 
00122   //zrange = 1:step(3):(sizez-winsize(3)+1);
00123   //yrange = 1:step(1):(sizey-winsize(1)+1);
00124   //xrange = 1:step(2):(sizex-winsize(2)+1);
00125 
00126   int ysize = (vrows-patopt.win.y) / patopt.step.y + 1;
00127   int xsize = (vcols-patopt.win.x) / patopt.step.x + 1;
00128   int maxsize = xsize*ysize;
00129   locyx.reserve(maxsize);
00130   locyx_mask.reserve(maxsize);
00131   allx.reserve(xsize);
00132   ally.reserve(ysize);
00133 
00134 #ifdef DEBUG
00135   cout << "x=" << xsize << ", y=" << ysize << " max=" << maxsize << "\n";
00136 #endif
00137 
00138   // locyx (iterate over x then y) 
00139   for (int y=0; y<=vrows-patopt.win.y; y+=patopt.step.y) {
00140     for (int x=0; x<=vcols-patopt.win.x; x+=patopt.step.x) {
00141       locyx.push_back(Point2i(x, y));
00142           locyx_mask.push_back(true);  //initialize mask
00143       if (y==0) allx.push_back(x);
00144     }
00145     ally.push_back(y);
00146   }
00147 
00148 #ifdef DEBUG
00149   cout << "p=" << locyx.size() << "\n";
00150   cout << "locyx = " << locyx << "\n";
00151   cout << "allx = ";
00152   for (int i=0; i<(int)allx.size(); i++)
00153     cout << allx[i] << " ";
00154   cout << "\n";
00155   cout << "ally = ";
00156   for (int i=0; i<(int)ally.size(); i++)
00157     cout << ally[i] << " ";
00158   cout << "\n";
00159 #endif
00160 
00161 }
00162 
00163 void PatchExtractor::reset() {
00164   locz  = -patopt.step.z;
00165   curz  = 0;
00166   nextz = patopt.win.z;
00167   vbuf = 0.0;
00168   if (flag_zm)
00169     vbuf_zm = 0.0;
00170 }
00171 
00172 int PatchExtractor::framesToNextPatch() {
00173   return (nextz-curz);
00174 }
00175 
00176 
00177 bool PatchExtractor::addFrames(const Mat &invid) {
00178   // get number of frames
00179   int sizez, sizex, sizey, dims;
00180   if (invid.dims == 2) {
00181     dims = 2;
00182     sizez = 1;
00183     sizey = invid.size[0];
00184     sizex = invid.size[1];
00185   } else {
00186     sizez = invid.size[0];
00187     sizey = invid.size[1];
00188     sizex = invid.size[2];
00189     dims = 3;
00190   }
00191 
00192   Mat vid;
00193   if (sizex == vcols && sizey == vrows) {
00194     vid = invid;
00195   } else {
00196     if (sizex < vcols || sizey < vrows)
00197       CV_Error(-1, "video not large enough!");
00198     
00199     cout << "WARNING: addFrames: video is too large!  cropping.\n";
00200     vid = MatVid::subvid(invid, 0, sizez, 0, vrows, 0, vcols);
00201   }
00202 
00203   // check if we are adding too much
00204   if (curz+sizez > nextz+patopt.step.z)
00205     CV_Error(-1, "adding too many frames!");
00206 
00207   // pop frames from the front (only if we're not replacing the full buffer)
00208   if (sizez < patopt.win.z) {
00209     for (int z=sizez; z<patopt.win.z; z++) {
00210       Mat fb = MatVid::frame(vbuf, z);
00211       Mat ff = MatVid::frame(vbuf, z-sizez);
00212       fb.copyTo(ff);
00213       //cout << "push z: " << z << " --> " <<z-sizez << "\n";
00214     }
00215   }
00216 
00217   // add frames on the back
00218   Mat vtmp;
00219   if (dims == 2) {
00220     Mat fb = MatVid::frame(vbuf, patopt.win.z-1);
00221     if (vid.type() == OPT_MAT_TYPE) {
00222       // just copy
00223       vid.copyTo(fb);
00224     } else {
00225       // convert first
00226       vid.convertTo(vtmp, OPT_MAT_TYPE);
00227       vtmp.copyTo(fb);
00228     }
00229 
00230   } else {
00231     for (int z=0; z<sizez; z++) {
00232       Mat fb = MatVid::frame(vbuf, patopt.win.z-sizez+z);
00233       Mat vf = MatVid::frame(vid, z);
00234       if (vf.type() == OPT_MAT_TYPE) {
00235         // just copy
00236         vf.copyTo(fb);
00237       } else {
00238         // convert first
00239         vf.convertTo(vtmp, OPT_MAT_TYPE);
00240         vtmp.copyTo(fb);
00241       }
00242       //cout << "newframe " << z << " --> " << patopt.win.z-sizez+z << "\n";
00243     }
00244   }
00245   
00246   // update current z position
00247   curz += sizez;
00248     
00249   // check if we made a new patch
00250   if (curz >= nextz) {
00251     // if yes, then process it and return true
00252     processNextPatch();
00253     return true;
00254 
00255   } else {
00256 
00257     // no new patch yet
00258     return false;
00259   }
00260 }
00261 
00262 
00263 
00264 // process patches
00265 void PatchExtractor::processNextPatch() {
00266   // update locations
00267   locz  += patopt.step.z; // next locz of patches
00268   nextz += patopt.step.z; // next new patch
00269   
00270   //update location mask using minvar
00271   if(patopt.minvar!=0)
00272   {
00273         
00274         for (int k=0; k<(int)locyx.size(); k++) 
00275         {
00276           // get the patch from buffer
00277           Mat mypat = MatVid::subvid(vbuf, Range(0, patopt.win.z), 
00278                                        Range(locyx[k].y, locyx[k].y+patopt.win.y), 
00279                                        Range(locyx[k].x, locyx[k].x+patopt.win.x));
00280 
00281         /*  Mat test1;
00282           mypat.copyTo(test1);
00283           int t1=test1.size[0];
00284           int t2=test1.size[1];
00285           int t3=test1.size[2];
00286           mypat=test1;*/
00287 
00288 
00289           //compute variance for each pixel
00290           std::vector<OPT_F_TYPE> pixvar;
00291           for(int i=0;i<mypat.size[1];i++)
00292                 for(int j=0;j<mypat.size[2];j++)
00293                 {                                                               
00294                         //Mat tmpMd=MatVid::subvid(mypat,Range::all(),Range(j,j+1),Range(i,i+1));                                                               
00295                         Mat tmpM=MatVid::subvid(mypat,Range(patopt.minvarf,patopt.win.z-patopt.minvarf),Range(j,j+1),Range(i,i+1));                                                             
00296 
00297                         //Mat tmpM=MatVid::subvid(mypat,Range(2,8),Range(j,j+1),Range(i,i+1));                                                          
00298                         //Mat tmpM=MatVid::subvid(mypat,Range(5,15),Range(j,j+1),Range(i,i+1));                                                         
00299                         //Mat tmpM=MatVid::subvid(mypat,Range(1,19),Range(j,j+1),Range(i,i+1)); 
00300 
00301                         OPT_F_TYPE var=computeVariance3dDouble(tmpM);
00302                         pixvar.push_back(var);                                                          
00303                 }                                                       
00304                 double mv,minvar2;
00305                 if(patopt.minvar>0)                             
00306                 {
00307                         mv=meanOfStdVector(pixvar);
00308                         minvar2=patopt.minvar;
00309                 }
00310                 else
00311                 {
00312                         mv=minOfStdVector(pixvar);
00313                         minvar2=-patopt.minvar;
00314                 }
00315                 //if below minimum, update the mask
00316                 if(mv<=minvar2)
00317                 {
00318                         locyx_mask[k]=false;                                    
00319                         
00320                 }
00321                 else
00322                 {
00323                         locyx_mask[k]=true;
00324                 }
00325    }
00326   }
00327 
00328   // process patches
00329   switch(patopt.normopt) {
00330   case PatchOptions::NORM_NONE:
00331     // do nothing
00332     break;
00333   case PatchOptions::NORM_ZM:
00334   case PatchOptions::NORM_ZMUV:
00335     {
00336       // compute the mean frame
00337       Mat Ymean;
00338       MatVid::reduce(vbuf, Ymean, CV_REDUCE_AVG);
00339       
00340       // subtract mean
00341       for (int z=0; z<patopt.win.z; z++) {
00342         Mat f   = MatVid::frame(vbuf, z);
00343         Mat fzm = MatVid::frame(vbuf_zm, z);
00344         fzm = f - Ymean;
00345       }      
00346       
00347       // also apply variance normalization
00348       if (patopt.normopt == PatchOptions::NORM_ZMUV) {
00349         // for each patch...
00350         for (int i=0; i<(int)locyx.size(); i++) {
00351           // get the patch in zero-mean buffer
00352           Mat zmpatch = MatVid::subvid(vbuf_zm, Range(0, patopt.win.z), 
00353                                        Range(locyx[i].y, locyx[i].y+patopt.win.y), 
00354                                        Range(locyx[i].x, locyx[i].x+patopt.win.x));
00355           
00356           // copy and normalize
00357           zmpatch.copyTo(patches[i]);
00358           double pstd = sqrt(MatVid::norm2(patches[i]) / (patches[i].total()-1.0));
00359           patches[i] *= (1.0/pstd);
00360         }
00361       }
00362     }
00363     break;
00364   default:
00365     CV_Error(-1, "bad option");
00366   }
00367 }
00368 
00369 
00370 // const so you can't clobber it
00371 const vector<Mat> & PatchExtractor::getPatches() {
00372   if (curz < patopt.win.z)
00373     CV_Error(-1, "not enough frames added yet!");
00374   if ( (curz-patopt.win.z) % patopt.step.z != 0)
00375     CV_Error(-1, "getPatch at wrong time!");
00376   
00377   return patches;
00378 }
00379 
00380 
00381 // get the center frame of the video buffer.
00382 const Mat & PatchExtractor::getCenterFrame() {
00383   return frame_center;
00384 }
00385   
00386 // return the frames (Mat video) associated with the current set of patches.
00387 const Mat & PatchExtractor::getPatchFrames() {
00388   return frame_patches;
00389 } 
00390 
00391 const Mat & PatchExtractor::getbuf() {
00392   return vbuf;
00393 }
00394 
00395 const Mat & PatchExtractor::getbuf_zm() {
00396   return vbuf_zm;
00397 }
00398 
00399 // conversion utilities
00400 
00401 // convert patch labels to an image (respects channels)
00402 // (same as segm_mask.m)
00403 void PatchExtractor::patchLabelsToImage(const Mat &patLabels, Mat &imgLabels, 
00404                                         fill_type fillMode) {
00405   bool filledges = false;
00406   Range validbox_x, validbox_y;  // the valid box, from which to fill edges
00407   
00408   // make labels into a row-vector (easier to process)
00409   Mat patLabelsV = patLabels.reshape(0, 1);
00410 
00411   // allocate destination image
00412   imgLabels.create(vrows, vcols, patLabels.type());
00413 
00414   // select the fill mode
00415   switch(fillMode) {
00416   case FILL_NN:
00417     {
00418       // create box around patch center
00419       int rstepx = patopt.step.x/2;
00420       int rstepy = patopt.step.y/2;
00421       Range box_x(-rstepx, patopt.step.x-rstepx);
00422       Range box_y(-rstepy, patopt.step.y-rstepy);
00423 
00424       for (unsigned int i=0; i<locyx.size(); i++) {
00425         // center of patch i
00426         int ty = locyx[i].y + coff.y;
00427         int tx = locyx[i].x + coff.x;
00428         
00429         // box for this patch
00430         Range mybox_x = box_x + tx;
00431         Range mybox_y = box_y + ty;
00432 
00433         // clip box to image
00434         if (mybox_x.start < 0)    mybox_x.start = 0;
00435         if (mybox_x.end >= vcols) mybox_x.end   = vcols;
00436         if (mybox_y.start < 0)    mybox_y.start = 0;
00437         if (mybox_y.end >= vrows) mybox_y.end   = vrows;
00438         
00439         // fill the patch with the label value
00440         Mat imgPat(imgLabels, mybox_y, mybox_x);
00441         repeat(patLabelsV.col(i), mybox_y.size(), mybox_x.size(), imgPat);
00442       }
00443       
00444       // flag to fill in the edges later
00445       filledges = true;
00446       validbox_x = Range(allx.front() + coff.x + box_x.start, 
00447                          allx.back()  + coff.x + box_x.end);
00448       validbox_y = Range(ally.front() + coff.y + box_y.start, 
00449                          ally.back()  + coff.y + box_y.end);
00450     }
00451     break;
00452 
00453   default:
00454     CV_Error(-1, "other fills not implemented");
00455   }
00456 
00457   // fill in the edges
00458   if (filledges) {
00459     int bordertop = (validbox_y.start > 0    ? validbox_y.start : 0);
00460     int borderbot = (validbox_y.end <= vrows ? vrows-validbox_y.end : 0);
00461     int borderleft = (validbox_x.start > 0    ? validbox_x.start : 0);
00462     int borderright = (validbox_x.end <= vcols ? vcols-validbox_x.end : 0);
00463     
00464     /*
00465     cout << "validbox_x = " << validbox_x << "\n";
00466     cout << "validbox_y = " << validbox_y << "\n";
00467     cout << "border[t,b,l,r] = [" << bordertop << " " << borderbot << " "
00468          << borderleft << " " << borderright << "]\n";
00469     */
00470 
00471     copyMakeBorder(Mat(imgLabels, validbox_y, validbox_x), 
00472                    imgLabels, 
00473                    bordertop, borderbot, borderleft, borderright,
00474                    BORDER_REPLICATE, 127);
00475 
00476     // make sure we didn't clobber the image
00477     CV_Assert(vrows == imgLabels.rows);
00478     CV_Assert(vcols == imgLabels.cols);
00479 
00480   }
00481 }
00482 
00483 
00484 // debugging
00485 void PatchExtractor::dump(ostream &output) const {
00486   output << "PatchExtractor (" << patopt << "; ";
00487   output << "vidsize=[" << vrows << "x" << vcols << "]; ";
00488   output << "locz=" << locz << "; ";
00489   output << "curz=" << curz << "; ";
00490   output << "nextz=" << nextz << "; ";
00491   output << "size(loc)=" << locyx.size() << "; ";
00492   output << "coff=[" << coff.y << "," << coff.x << "," << coff.z << "]";
00493   output << ")";
00494 }
00495 
00496 
00497 void PatchExtractor::dump_matlab_init(ostream &output) const {
00498   output << "clear pat;\n";
00499   output << "pat.winsize = " << patopt.win << ";\n";
00500   output << "pat.step = " << patopt.step << ";\n";
00501   output << "pat.coff = " << coff << ";\n";
00502   output << "pat.xrange = " << Mat(allx) << "'+1;\n";
00503   output << "pat.yrange = " << Mat(ally) << "'+1;\n"; 
00504   output << "pat.zrange = [];\n";
00505   output << "pat.ipatch = {};\n";
00506   output << "pat.loc = [];\n";
00507 }
00508 
00509 void PatchExtractor::dump_matlab_patches(ostream &output) const {
00510   for (unsigned int i=0; i<patches.size(); i++) {
00511     output << "pat.ipatch{end+1} = ";
00512     MatVid::dump_matlab(patches[i], output);
00513     output << ";\n";
00514     
00515     output << "pat.loc(end+1,:) = [" << locyx[i].y << ","
00516            << locyx[i].x << "," << locz << "]+1;\n";
00517   }   
00518   output << "pat.zrange(end+1) = " << locz << "+1;\n";      
00519 }
00520 
00521 // convert to string
00522 ostream& operator<<(ostream& output, const PatchExtractor& pat) {
00523   pat.dump(output);
00524   return output;
00525 }

Generated on 18 Dec 2014 for libdt-OpenCV by  doxygen 1.6.1