00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013 #include <iostream>
00014 #include <string>
00015 #include<fstream>
00016
00017 #include "opencv/cv.h"
00018 #include "opencv/cxcore.h"
00019 #include "opencv/highgui.h"
00020
00021 #include "options.h"
00022 #include "utils/libutils.h"
00023 #include "video/PatchOptions.hpp"
00024 #include "video/PatchExtractor.hpp"
00025 #include "video/MatVid.hpp"
00026
00027
00028
00029
00030
00031
00032
00033
00034
00035
00036
00037
00038 PatchExtractor::PatchExtractor(const PatchOptions & patopt, int vrows, int vcols) :
00039 patopt(patopt), vrows(vrows), vcols(vcols)
00040 {
00041 CV_Assert((vrows>=patopt.win.y) && (vcols>=patopt.win.x));
00042
00043
00044 getLoc();
00045
00046
00047 vbuf = MatVid::create(patopt.win.z, vrows, vcols, OPT_MAT_TYPE);
00048 if (patopt.normopt != PatchOptions::NORM_NONE) {
00049 vbuf_zm = MatVid::create(patopt.win.z, vrows, vcols, OPT_MAT_TYPE);
00050 flag_zm = true;
00051 } else {
00052 flag_zm = false;
00053 }
00054
00055
00056 patches.resize(locyx.size());
00057 switch(patopt.normopt) {
00058 case PatchOptions::NORM_NONE:
00059
00060 for (int i=0; i<(int)locyx.size(); i++) {
00061 patches[i] = MatVid::subvid(vbuf, Range(0, patopt.win.z),
00062 Range(locyx[i].y, locyx[i].y+patopt.win.y),
00063 Range(locyx[i].x, locyx[i].x+patopt.win.x));
00064 }
00065 break;
00066 case PatchOptions::NORM_ZM:
00067
00068 for (int i=0; i<(int)locyx.size(); i++) {
00069 patches[i] = MatVid::subvid(vbuf_zm, Range(0, patopt.win.z),
00070 Range(locyx[i].y, locyx[i].y+patopt.win.y),
00071 Range(locyx[i].x, locyx[i].x+patopt.win.x));
00072 }
00073 break;
00074 case PatchOptions::NORM_ZMUV:
00075
00076 for (int i=0; i<(int)locyx.size(); i++) {
00077 patches[i] = MatVid::create(patopt.win.z, patopt.win.y, patopt.win.x, OPT_MAT_TYPE);
00078 }
00079 break;
00080 default:
00081 CV_Error(-1, "bad option");
00082 }
00083
00084
00085 reset();
00086
00087
00088
00089 frame_center = MatVid::frame(vbuf, coff.z);
00090
00091
00092
00093
00094 int tmp = patopt.step.z / 2;
00095 frame_patches = MatVid::subvid(vbuf,
00096 Range(coff.z-tmp, coff.z-tmp+patopt.step.z),
00097 Range::all(), Range::all());
00098
00099 }
00100
00101
00102 PatchExtractor::~PatchExtractor() {
00103 }
00104
00105
00106 void PatchExtractor::getLoc() {
00107
00108 #ifdef DEBUG
00109 cout << "size: " << vrows << "," << vcols << "\n";
00110 #endif
00111
00112
00113
00114
00115
00116
00117 coff.x=patopt.win.x*0.5;
00118 coff.y=patopt.win.y*0.5;
00119 coff.z=patopt.win.z*0.5;
00120
00121
00122
00123
00124
00125
00126 int ysize = (vrows-patopt.win.y) / patopt.step.y + 1;
00127 int xsize = (vcols-patopt.win.x) / patopt.step.x + 1;
00128 int maxsize = xsize*ysize;
00129 locyx.reserve(maxsize);
00130 locyx_mask.reserve(maxsize);
00131 allx.reserve(xsize);
00132 ally.reserve(ysize);
00133
00134 #ifdef DEBUG
00135 cout << "x=" << xsize << ", y=" << ysize << " max=" << maxsize << "\n";
00136 #endif
00137
00138
00139 for (int y=0; y<=vrows-patopt.win.y; y+=patopt.step.y) {
00140 for (int x=0; x<=vcols-patopt.win.x; x+=patopt.step.x) {
00141 locyx.push_back(Point2i(x, y));
00142 locyx_mask.push_back(true);
00143 if (y==0) allx.push_back(x);
00144 }
00145 ally.push_back(y);
00146 }
00147
00148 #ifdef DEBUG
00149 cout << "p=" << locyx.size() << "\n";
00150 cout << "locyx = " << locyx << "\n";
00151 cout << "allx = ";
00152 for (int i=0; i<(int)allx.size(); i++)
00153 cout << allx[i] << " ";
00154 cout << "\n";
00155 cout << "ally = ";
00156 for (int i=0; i<(int)ally.size(); i++)
00157 cout << ally[i] << " ";
00158 cout << "\n";
00159 #endif
00160
00161 }
00162
00163 void PatchExtractor::reset() {
00164 locz = -patopt.step.z;
00165 curz = 0;
00166 nextz = patopt.win.z;
00167 vbuf = 0.0;
00168 if (flag_zm)
00169 vbuf_zm = 0.0;
00170 }
00171
00172 int PatchExtractor::framesToNextPatch() {
00173 return (nextz-curz);
00174 }
00175
00176
00177 bool PatchExtractor::addFrames(const Mat &invid) {
00178
00179 int sizez, sizex, sizey, dims;
00180 if (invid.dims == 2) {
00181 dims = 2;
00182 sizez = 1;
00183 sizey = invid.size[0];
00184 sizex = invid.size[1];
00185 } else {
00186 sizez = invid.size[0];
00187 sizey = invid.size[1];
00188 sizex = invid.size[2];
00189 dims = 3;
00190 }
00191
00192 Mat vid;
00193 if (sizex == vcols && sizey == vrows) {
00194 vid = invid;
00195 } else {
00196 if (sizex < vcols || sizey < vrows)
00197 CV_Error(-1, "video not large enough!");
00198
00199 cout << "WARNING: addFrames: video is too large! cropping.\n";
00200 vid = MatVid::subvid(invid, 0, sizez, 0, vrows, 0, vcols);
00201 }
00202
00203
00204 if (curz+sizez > nextz+patopt.step.z)
00205 CV_Error(-1, "adding too many frames!");
00206
00207
00208 if (sizez < patopt.win.z) {
00209 for (int z=sizez; z<patopt.win.z; z++) {
00210 Mat fb = MatVid::frame(vbuf, z);
00211 Mat ff = MatVid::frame(vbuf, z-sizez);
00212 fb.copyTo(ff);
00213
00214 }
00215 }
00216
00217
00218 Mat vtmp;
00219 if (dims == 2) {
00220 Mat fb = MatVid::frame(vbuf, patopt.win.z-1);
00221 if (vid.type() == OPT_MAT_TYPE) {
00222
00223 vid.copyTo(fb);
00224 } else {
00225
00226 vid.convertTo(vtmp, OPT_MAT_TYPE);
00227 vtmp.copyTo(fb);
00228 }
00229
00230 } else {
00231 for (int z=0; z<sizez; z++) {
00232 Mat fb = MatVid::frame(vbuf, patopt.win.z-sizez+z);
00233 Mat vf = MatVid::frame(vid, z);
00234 if (vf.type() == OPT_MAT_TYPE) {
00235
00236 vf.copyTo(fb);
00237 } else {
00238
00239 vf.convertTo(vtmp, OPT_MAT_TYPE);
00240 vtmp.copyTo(fb);
00241 }
00242
00243 }
00244 }
00245
00246
00247 curz += sizez;
00248
00249
00250 if (curz >= nextz) {
00251
00252 processNextPatch();
00253 return true;
00254
00255 } else {
00256
00257
00258 return false;
00259 }
00260 }
00261
00262
00263
00264
00265 void PatchExtractor::processNextPatch() {
00266
00267 locz += patopt.step.z;
00268 nextz += patopt.step.z;
00269
00270
00271 if(patopt.minvar!=0)
00272 {
00273
00274 for (int k=0; k<(int)locyx.size(); k++)
00275 {
00276
00277 Mat mypat = MatVid::subvid(vbuf, Range(0, patopt.win.z),
00278 Range(locyx[k].y, locyx[k].y+patopt.win.y),
00279 Range(locyx[k].x, locyx[k].x+patopt.win.x));
00280
00281
00282
00283
00284
00285
00286
00287
00288
00289
00290 std::vector<OPT_F_TYPE> pixvar;
00291 for(int i=0;i<mypat.size[1];i++)
00292 for(int j=0;j<mypat.size[2];j++)
00293 {
00294
00295 Mat tmpM=MatVid::subvid(mypat,Range(patopt.minvarf,patopt.win.z-patopt.minvarf),Range(j,j+1),Range(i,i+1));
00296
00297
00298
00299
00300
00301 OPT_F_TYPE var=computeVariance3dDouble(tmpM);
00302 pixvar.push_back(var);
00303 }
00304 double mv,minvar2;
00305 if(patopt.minvar>0)
00306 {
00307 mv=meanOfStdVector(pixvar);
00308 minvar2=patopt.minvar;
00309 }
00310 else
00311 {
00312 mv=minOfStdVector(pixvar);
00313 minvar2=-patopt.minvar;
00314 }
00315
00316 if(mv<=minvar2)
00317 {
00318 locyx_mask[k]=false;
00319
00320 }
00321 else
00322 {
00323 locyx_mask[k]=true;
00324 }
00325 }
00326 }
00327
00328
00329 switch(patopt.normopt) {
00330 case PatchOptions::NORM_NONE:
00331
00332 break;
00333 case PatchOptions::NORM_ZM:
00334 case PatchOptions::NORM_ZMUV:
00335 {
00336
00337 Mat Ymean;
00338 MatVid::reduce(vbuf, Ymean, CV_REDUCE_AVG);
00339
00340
00341 for (int z=0; z<patopt.win.z; z++) {
00342 Mat f = MatVid::frame(vbuf, z);
00343 Mat fzm = MatVid::frame(vbuf_zm, z);
00344 fzm = f - Ymean;
00345 }
00346
00347
00348 if (patopt.normopt == PatchOptions::NORM_ZMUV) {
00349
00350 for (int i=0; i<(int)locyx.size(); i++) {
00351
00352 Mat zmpatch = MatVid::subvid(vbuf_zm, Range(0, patopt.win.z),
00353 Range(locyx[i].y, locyx[i].y+patopt.win.y),
00354 Range(locyx[i].x, locyx[i].x+patopt.win.x));
00355
00356
00357 zmpatch.copyTo(patches[i]);
00358 double pstd = sqrt(MatVid::norm2(patches[i]) / (patches[i].total()-1.0));
00359 patches[i] *= (1.0/pstd);
00360 }
00361 }
00362 }
00363 break;
00364 default:
00365 CV_Error(-1, "bad option");
00366 }
00367 }
00368
00369
00370
00371 const vector<Mat> & PatchExtractor::getPatches() {
00372 if (curz < patopt.win.z)
00373 CV_Error(-1, "not enough frames added yet!");
00374 if ( (curz-patopt.win.z) % patopt.step.z != 0)
00375 CV_Error(-1, "getPatch at wrong time!");
00376
00377 return patches;
00378 }
00379
00380
00381
00382 const Mat & PatchExtractor::getCenterFrame() {
00383 return frame_center;
00384 }
00385
00386
00387 const Mat & PatchExtractor::getPatchFrames() {
00388 return frame_patches;
00389 }
00390
00391 const Mat & PatchExtractor::getbuf() {
00392 return vbuf;
00393 }
00394
00395 const Mat & PatchExtractor::getbuf_zm() {
00396 return vbuf_zm;
00397 }
00398
00399
00400
00401
00402
00403 void PatchExtractor::patchLabelsToImage(const Mat &patLabels, Mat &imgLabels,
00404 fill_type fillMode) {
00405 bool filledges = false;
00406 Range validbox_x, validbox_y;
00407
00408
00409 Mat patLabelsV = patLabels.reshape(0, 1);
00410
00411
00412 imgLabels.create(vrows, vcols, patLabels.type());
00413
00414
00415 switch(fillMode) {
00416 case FILL_NN:
00417 {
00418
00419 int rstepx = patopt.step.x/2;
00420 int rstepy = patopt.step.y/2;
00421 Range box_x(-rstepx, patopt.step.x-rstepx);
00422 Range box_y(-rstepy, patopt.step.y-rstepy);
00423
00424 for (unsigned int i=0; i<locyx.size(); i++) {
00425
00426 int ty = locyx[i].y + coff.y;
00427 int tx = locyx[i].x + coff.x;
00428
00429
00430 Range mybox_x = box_x + tx;
00431 Range mybox_y = box_y + ty;
00432
00433
00434 if (mybox_x.start < 0) mybox_x.start = 0;
00435 if (mybox_x.end >= vcols) mybox_x.end = vcols;
00436 if (mybox_y.start < 0) mybox_y.start = 0;
00437 if (mybox_y.end >= vrows) mybox_y.end = vrows;
00438
00439
00440 Mat imgPat(imgLabels, mybox_y, mybox_x);
00441 repeat(patLabelsV.col(i), mybox_y.size(), mybox_x.size(), imgPat);
00442 }
00443
00444
00445 filledges = true;
00446 validbox_x = Range(allx.front() + coff.x + box_x.start,
00447 allx.back() + coff.x + box_x.end);
00448 validbox_y = Range(ally.front() + coff.y + box_y.start,
00449 ally.back() + coff.y + box_y.end);
00450 }
00451 break;
00452
00453 default:
00454 CV_Error(-1, "other fills not implemented");
00455 }
00456
00457
00458 if (filledges) {
00459 int bordertop = (validbox_y.start > 0 ? validbox_y.start : 0);
00460 int borderbot = (validbox_y.end <= vrows ? vrows-validbox_y.end : 0);
00461 int borderleft = (validbox_x.start > 0 ? validbox_x.start : 0);
00462 int borderright = (validbox_x.end <= vcols ? vcols-validbox_x.end : 0);
00463
00464
00465
00466
00467
00468
00469
00470
00471 copyMakeBorder(Mat(imgLabels, validbox_y, validbox_x),
00472 imgLabels,
00473 bordertop, borderbot, borderleft, borderright,
00474 BORDER_REPLICATE, 127);
00475
00476
00477 CV_Assert(vrows == imgLabels.rows);
00478 CV_Assert(vcols == imgLabels.cols);
00479
00480 }
00481 }
00482
00483
00484
00485 void PatchExtractor::dump(ostream &output) const {
00486 output << "PatchExtractor (" << patopt << "; ";
00487 output << "vidsize=[" << vrows << "x" << vcols << "]; ";
00488 output << "locz=" << locz << "; ";
00489 output << "curz=" << curz << "; ";
00490 output << "nextz=" << nextz << "; ";
00491 output << "size(loc)=" << locyx.size() << "; ";
00492 output << "coff=[" << coff.y << "," << coff.x << "," << coff.z << "]";
00493 output << ")";
00494 }
00495
00496
00497 void PatchExtractor::dump_matlab_init(ostream &output) const {
00498 output << "clear pat;\n";
00499 output << "pat.winsize = " << patopt.win << ";\n";
00500 output << "pat.step = " << patopt.step << ";\n";
00501 output << "pat.coff = " << coff << ";\n";
00502 output << "pat.xrange = " << Mat(allx) << "'+1;\n";
00503 output << "pat.yrange = " << Mat(ally) << "'+1;\n";
00504 output << "pat.zrange = [];\n";
00505 output << "pat.ipatch = {};\n";
00506 output << "pat.loc = [];\n";
00507 }
00508
00509 void PatchExtractor::dump_matlab_patches(ostream &output) const {
00510 for (unsigned int i=0; i<patches.size(); i++) {
00511 output << "pat.ipatch{end+1} = ";
00512 MatVid::dump_matlab(patches[i], output);
00513 output << ";\n";
00514
00515 output << "pat.loc(end+1,:) = [" << locyx[i].y << ","
00516 << locyx[i].x << "," << locz << "]+1;\n";
00517 }
00518 output << "pat.zrange(end+1) = " << locz << "+1;\n";
00519 }
00520
00521
00522 ostream& operator<<(ostream& output, const PatchExtractor& pat) {
00523 pat.dump(output);
00524 return output;
00525 }