Dytex.cpp

00001 /**************************************************
00002 
00003   Dytex class
00004 
00005   Antoni Chan,
00006   City University of Hong Kong, Copyright 2014
00007   libdt - OpenCV library for Dynamic Textures - version 1.0
00008     
00009 ***************************************************/
00010 
00011 #include <iostream>
00012 #include <string>
00013 #include<fstream>
00014 
00015 #include "opencv/cv.h"
00016 #include "opencv/cxcore.h"
00017 #include "opencv/highgui.h"
00018 
00019 
00020 #include "options.h"
00021 #include "utils/libutils.h"
00022 #include "video/libvideo.h"
00023 #include "stats/libstats.h"
00024 #include "dytex/DytexOptions.hpp"
00025 #include "dytex/DytexRegOptions.hpp"
00026 #include "dytex/Dytex.hpp"
00027 
00028 // constructor
00029 Dytex::Dytex()
00030 {
00031         isempty=true;
00032 }
00033 
00034 // constructor
00035 Dytex::Dytex(const DytexOptions &opt) :
00036   dtopt(opt), 
00037   Ymean( (dtopt.Yopt == DytexOptions::NONZERO_YMEAN ? dtopt.m : 0), 
00038          (dtopt.Yopt == DytexOptions::NONZERO_YMEAN ? 1 : 0), OPT_MAT_TYPE), 
00039   A(dtopt.n, dtopt.n, OPT_MAT_TYPE),
00040   C(dtopt.m, dtopt.n, OPT_MAT_TYPE),
00041   mu0(dtopt.n, 1, OPT_MAT_TYPE),
00042   R(dtopt.m, dtopt.Ropt),
00043   Q(dtopt.n, CovMatrix::COV_FULL),
00044   S0(dtopt.n, dtopt.Sopt),
00045   vrows(0), vcols(0)
00046 {
00047         isempty=false;
00048         isCvs=false;
00049   switch(dtopt.Yopt) {
00050   case DytexOptions::NONZERO_YMEAN:
00051   case DytexOptions::ZERO_YMEAN:
00052     break;
00053   default:
00054     CV_Error(-1, "bad Yopt");
00055   }
00056 }
00057 
00058 
00059 // destructor
00060 Dytex::~Dytex() {
00061   // do nothing
00062 }
00063 
00064 // set regularization
00065 void Dytex::setRegularizer(const DytexRegOptions &dtreg) {
00066   R.setRegularizer(dtreg.Ropt, dtreg.Rval);
00067   Q.setRegularizer(dtreg.Qopt, dtreg.Qval);
00068   S0.setRegularizer(dtreg.Sopt, dtreg.Sval);
00069 }
00070 
00071 void Dytex::regularize(bool regA) 
00072 {
00073   R.regularize();
00074   Q.regularize();
00075   S0.regularize();
00076 
00077   if(regA) //For HEM
00078   {
00079           //Regularization of A         
00080           double target=0.999;
00081           Mat E;
00082 
00083           E=eigc(A);
00084           Mat Eabs=eigc_abs(E);
00085           double minVal,maxVal;
00086           minMaxLoc(Eabs,&minVal,&maxVal);
00087           if(maxVal>target)
00088           {
00089                   A=A*target/maxVal;
00090           }
00091   }
00092 }
00093 
00094 void Dytex::regularizeA() 
00095 {
00096   
00097         //Regularization of A           
00098         double target=0.999;
00099         Mat E;
00100 
00101         E=eigc(A);
00102         Mat Eabs=eigc_abs(E);
00103         double minVal,maxVal;
00104         minMaxLoc(Eabs,&minVal,&maxVal);
00105         if(maxVal>target)
00106         {
00107                 A=A*target/maxVal;
00108         }
00109  
00110 }
00111 
00112 // convert a video into a vector time-series
00113 // also subtract the empirical mean if necessary, and return it.
00114 void Dytex::processVideoTraining(const Mat &vin, Mat &Yout, Mat &Yout_mean) {
00115   Mat Ytmp;
00116   
00117   switch(vin.dims) {
00118   case 3:
00119     // vectorize
00120     Ytmp = MatVid::vectorize(vin, true);
00121     
00122     // save/check video size
00123     if (vrows == 0 && vcols == 0) {
00124       vrows = vin.size[1];
00125       vcols = vin.size[2];
00126     } else {
00127       CV_Assert(vrows == vin.size[1] && vcols == vin.size[2]);
00128     }
00129     break;
00130     
00131   case 2:
00132     // already vectorized
00133     Ytmp = vin;
00134     break;
00135     
00136   default:
00137     CV_Error(-1, "Unsupported video dims (color)");
00138   }
00139   
00140   //cout << "Ytmp: "; dumpMatSize(Ytmp);
00141 
00142   // --- check dimensions ---
00143   //CV_Assert(Ytmp.rows == Yout.rows);
00144   //CV_Assert(Ytmp.cols == Yout.cols);
00145     
00146   // --- convert to double/float ---
00147   if (Ytmp.type() != OPT_MAT_TYPE) {
00148     //cout << "llS convertTo float\n";
00149     Ytmp.convertTo(Yout, OPT_MAT_TYPE); 
00150   } else {
00151     Ytmp.copyTo(Yout); // copy to Yall
00152   }
00153     
00154   // --- compute the mean for the video ---
00155   switch (dtopt.Yopt) {
00156   case DytexOptions::NONZERO_YMEAN:
00157     // compute the mean
00158     reduce(Yout, Yout_mean, 1, CV_REDUCE_AVG);
00159     // subtract mean from Y
00160     for (int t=0; t<Yout.cols; t++)
00161       Yout.col(t) -= Yout_mean;
00162     break;
00163     
00164   case DytexOptions::ZERO_YMEAN:
00165     // do nothing
00166     break;
00167   default:
00168     CV_Error(-1, "bad Yopt");
00169   }
00170 }
00171 
00172 // convert a video into a vector time-series, subtract the Ymean in the dytex
00173 void Dytex::processVideoTesting(const Mat &vin, Mat &Yout) const {
00174   Mat Ytmp;
00175   
00176   switch(vin.dims) {
00177   case 3:
00178     // check video size
00179     if (!(vrows == 0 && vcols == 0))
00180       CV_Assert(vrows == vin.size[1] && vcols == vin.size[2]);
00181     // vectorize
00182     Ytmp = MatVid::vectorize(vin, true);
00183     break;
00184     
00185   case 2:
00186     // already vectorized
00187     Ytmp = vin;
00188     break;
00189     
00190   default:
00191     CV_Error(-1, "Unsupported video dims (color)");
00192   }
00193   
00194   //cout << "Ytmp: "; dumpMatSize(Ytmp);
00195 
00196   // --- check dimensions ---
00197   //CV_Assert(Ytmp.rows == Yout.rows);
00198   //CV_Assert(Ytmp.cols == Yout.cols);
00199     
00200   // --- convert to double/float ---
00201   if (Ytmp.type() != OPT_MAT_TYPE) {
00202     //cout << "llS convertTo float\n";
00203     Ytmp.convertTo(Yout, OPT_MAT_TYPE); 
00204   } else {
00205     Ytmp.copyTo(Yout); // copy to Yall
00206   }
00207     
00208   // --- subtract mean ---
00209   switch (dtopt.Yopt) {
00210   case DytexOptions::NONZERO_YMEAN:
00211     // subtract mean from Y
00212     for (int t=0; t<Yout.cols; t++)
00213       Yout.col(t) -= Ymean;
00214     break;
00215     
00216   case DytexOptions::ZERO_YMEAN:
00217     // do nothing
00218     break;
00219   default:
00220     CV_Error(-1, "bad Yopt");
00221   }
00222 }
00223 
00224 
00225 // learning (multiple videos)
00226 void Dytex::learnLeastSquares(const vector<Mat> &videos) {
00227   // --- process video ---
00228   int nump = videos.size(); // number of videos
00229   int* tau  = (int*) malloc( sizeof(int) * nump );                              // length of each video
00230 
00231   int* inds = (int*) malloc( sizeof(int) * ( nump + 1 ) );              // index boundaries for each video
00232 
00233   //int tau[nump];      // length of each video
00234   //int inds[nump+1];   // index boundaries for each video
00235   int tauall;
00236 
00237   inds[0] = 0;
00238 
00239   // figure out the size of all videos
00240   for (int p=0; p<nump; p++) {
00241     int mytau = 0;
00242     if (videos[p].dims == 3)
00243       mytau = videos[p].size[0];
00244     else if (videos[p].dims == 2)
00245       mytau = videos[p].cols;
00246     else
00247       CV_Error(-1, "unknown size");
00248     
00249     tau[p]    = mytau;
00250     inds[p+1] = inds[p]+mytau;    
00251   }
00252   tauall = inds[nump];
00253 
00254   //cout << "tau = "; for (int p=0; p<nump; p++) cout << tau[p] << " "; cout << "\n";
00255   //cout << "inds = "; for (int p=0; p<nump; p++)  cout << inds[p] << " "; cout << "\n";
00256 
00257   // allocate space
00258   Mat Yall(dtopt.m, tauall, OPT_MAT_TYPE);
00259   Mat tmp1;
00260   
00261   //cout << "Yall: "; dumpMatSize(Yall);
00262 
00263   // initialize Ymean accumulator
00264   if (dtopt.Yopt == DytexOptions::NONZERO_YMEAN)
00265     Ymean = 0.0;
00266 
00267   // for each video, process it, compute and subtract the mean
00268   for (int p=0; p<nump; p++) {
00269     Mat Yall_p = Yall.colRange(inds[p], inds[p+1]); // get video p in Yall
00270     //cout << "Yall_p: "; dumpMatSize(Yall_p);
00271 
00272     processVideoTraining(videos[p], Yall_p, tmp1);
00273 
00274     // update accumulator
00275     if (dtopt.Yopt == DytexOptions::NONZERO_YMEAN)
00276       Ymean += tmp1;
00277   }
00278 
00279   // normalize mean
00280   if (dtopt.Yopt == DytexOptions::NONZERO_YMEAN) {
00281     Ymean /= nump;
00282   }
00283 
00284   // --- special case when n=0 ---
00285   if (dtopt.n == 0) {
00286     // only need to calculate R
00287     R.calculate(Yall, true);
00288         free(tau);
00289         free(inds);
00290     return;
00291   }
00292 
00293   // --- do PCA on Yall ---
00294   Mat Xall;
00295   if (Yall.rows > Yall.cols) {
00296     // ... using SVD
00297     //cout << "lls:use SVD\n";
00298     SVD svd(Yall);
00299     svd.u.colRange(0,dtopt.n).copyTo(C);  // C = u(1:n)
00300     
00301     //cout << "singular values: " << svd.w << "\n";
00302 
00303     // X = w(1:n)*vt(1:n,:)
00304     svd.vt.rowRange(0, dtopt.n).copyTo(Xall);
00305     Mat tmpw = svd.w.rowRange(0, dtopt.n);
00306     scaleRows(Xall, tmpw);
00307 
00308     //Mat Xall2 = C.t()*Yall;   // X = C'*Yall    
00309     //cout << "Xall err=" << norm(Xall, Xall2) << "\n";
00310 
00311   } else {
00312     // ... using eig
00313     //cout << "lls:use eig\n";
00314 
00315     Mat tmp;
00316     mulTransposed(Yall, tmp, false);
00317     SVD svd(tmp);
00318     svd.u.colRange(0,dtopt.n).copyTo(C);  // C = u(1:n)
00319     
00320     //cout << "singular values: " << svd.w << "\n";
00321 
00322     Xall = C.t()*Yall;   // X = C'*Yall
00323   }
00324 
00325   // --- estimate initial condition ---
00326   if (nump == 1) {
00327     // only one video (just copy it)
00328     Xall.col(0).copyTo(mu0);
00329     S0.mtx = 0.0;
00330 
00331   } else {
00332     // extract first X from each image and store in Xall0
00333     Mat Xall0(dtopt.n, nump, OPT_MAT_TYPE);
00334     for (int p=0; p<nump; p++) {
00335       Mat Xall0c = Xall0.col(p);
00336       Xall.col(inds[p]).copyTo(Xall0c);
00337     }
00338     // estimate the initial mean
00339     reduce(Xall0, mu0, 1, CV_REDUCE_AVG);
00340     // estimate the initial covariance
00341     for (int p=0; p<nump; p++)
00342       Xall0.col(p) -= mu0;     // subtract inital mean
00343     S0.calculate(Xall0, true); // calculate covariance
00344   }
00345   
00346   // --- estimate A ---
00347   Mat Phi1all; // store X(1:end-1)
00348   Mat Phi2all; // store X(2:end)
00349 
00350   if (nump == 1) {
00351     // only one video, just make pointers (no memory copied)
00352     Phi1all = Xall.colRange(0, Xall.cols-1);
00353     Phi2all = Xall.colRange(1, Xall.cols);
00354 
00355   } else {
00356     // copy to Phi1all and Phi2all
00357     // allocate space
00358     Phi1all.create(dtopt.n, tauall-nump, OPT_MAT_TYPE); // store X(1:end-1)
00359     Phi2all.create(dtopt.n, tauall-nump, OPT_MAT_TYPE); // store X(2:end)
00360 
00361     //cout << "Phi1all: "; dumpMatSize(Phi1all);
00362     //cout << "Phi2all: "; dumpMatSize(Phi2all);
00363 
00364     // for each video
00365     for (int p=0; p<nump; p++) {
00366       Mat Phi1 = Xall.colRange(inds[p], inds[p+1]-1);
00367       Mat pPhi1all = Phi1all.colRange(inds[p]-p, inds[p+1]-(p+1));
00368       //cout << "Phi1: "; dumpMatSize(Phi1);
00369       //cout << "pPhi1all: "; dumpMatSize(pPhi1all);
00370       Phi1.copyTo(pPhi1all);
00371       
00372       Mat Phi2 = Xall.colRange(inds[p]+1, inds[p+1]);
00373       Mat pPhi2all = Phi2all.colRange(inds[p]-p, inds[p+1]-(p+1));
00374       //cout << "Phi2: "; dumpMatSize(Phi2);
00375       //cout << "pPhi2all: "; dumpMatSize(pPhi2all);
00376       Phi2.copyTo(pPhi2all);
00377     }
00378   }
00379 
00380   Mat iPhi1all;
00381   invert(Phi1all, iPhi1all, DECOMP_SVD);
00382   A = Phi2all*iPhi1all;
00383   
00384   // --- estimate Q ---
00385   Mat V = Phi2all - A*Phi1all;
00386   Q.calculate(V, false);
00387   
00388   // --- estimate R ---
00389   Mat errory = Yall - C*Xall;
00390   R.calculate(errory, true);   // (to match matlab)
00391 
00392   // clean up
00393   free(tau);
00394   free(inds);
00395 }
00396 
00397 // learning (single video)
00398 void Dytex::learnLeastSquares(const Mat &video) {
00399   vector<Mat> videos(1);
00400   videos[0] = video;
00401   learnLeastSquares(videos);
00402 }
00403 
00404 
00405 /*
00406 // learning (single video)
00407 void Dytex::learnLeastSquares_OLD(const Mat &video) {
00408   // --- process video ---
00409   Mat Yall;
00410   bool iscopy = false;
00411   {
00412     Mat Ytmp;
00413     if (video.dims == 3) {
00414       // vectorize
00415       Ytmp = MatVid::vectorize(video, true);
00416       vrows = video.size[1];
00417       vcols = video.size[2];
00418       iscopy = true;
00419       
00420     } else if (video.dims == 2) {
00421       // already vectorized
00422       Ytmp = video;
00423       iscopy = false;
00424       
00425     } else {
00426       CV_Error(-1, "Unsupported video dims (color)");
00427     }
00428     
00429     // --- convert to double/float ---
00430     if (Ytmp.type() != OPT_MAT_TYPE) {
00431       //cout << "llS convertTo float\n";
00432       Ytmp.convertTo(Yall, OPT_MAT_TYPE);
00433     } else {
00434       if (iscopy)
00435         Yall = Ytmp; // do nothing
00436       else
00437         Ytmp.copyTo(Yall); // make a copy for processing later
00438     }
00439   }
00440   
00441   // --- check dimensions ---
00442   CV_Assert(Yall.rows == dtopt.m);
00443 
00444   // --- compute the mean ---
00445   switch (dtopt.Yopt) {
00446   case DytexOptions::NONZERO_YMEAN:
00447     // compute the mean
00448     reduce(Yall, Ymean, 1, CV_REDUCE_AVG);
00449     // subtract mean from Y
00450     for (int t=0; t<Yall.cols; t++)
00451       Yall.col(t) -= Ymean;
00452     break;
00453 
00454   case DytexOptions::ZERO_YMEAN:
00455     // do nothing
00456     break;
00457   default:
00458     CV_Error(-1, "bad Yopt");
00459   }
00460 
00461   // --- special case when n=0 ---
00462   if (dtopt.n == 0) {
00463     // only need to calculate R
00464     R.calculate(Yall, true);
00465     return;
00466   }
00467 
00468   // --- do PCA on Yall ---
00469   Mat Xall;
00470   if (Yall.rows > Yall.cols) {
00471     // ... using SVD
00472     cout << "lls:use SVD\n";
00473     SVD svd(Yall);
00474     svd.u.colRange(0,dtopt.n).copyTo(C);  // C = u(1:n)
00475     
00476     //cout << "singular values: " << svd.w << "\n";
00477 
00478     Xall = C.t()*Yall;   // X = C'*Yall
00479     // TODO: can also do w(1:n)*vt(1:n,:)
00480 
00481   } else {
00482     // ... using eig
00483     cout << "lls:use eig\n";
00484 
00485     Mat tmp;
00486     mulTransposed(Yall, tmp, false);
00487     SVD svd(tmp);
00488     svd.u.colRange(0,dtopt.n).copyTo(C);  // C = u(1:n)
00489     
00490     cout << "singular values: " << svd.w << "\n";
00491 
00492     Xall = C.t()*Yall;   // X = C'*Yall
00493   }
00494 
00495   // --- estimate initial condition ---
00496   Xall.col(0).copyTo(mu0);
00497   S0.mtx = 0.0;
00498   
00499   // --- estimate A ---
00500   Mat Phi1 = Xall.colRange(0, Xall.cols-1);
00501   Mat Phi2 = Xall.colRange(1, Xall.cols);
00502   Mat iPhi1;
00503   
00504   invert(Phi1, iPhi1, DECOMP_SVD);
00505   A = Phi2*iPhi1;
00506   
00507   // --- estimate Q ---
00508   Mat V = Phi2 - A*Phi1;
00509   Q.calculate(V, false);
00510   
00511   // --- estimate R ---
00512   Mat errory = Yall - C*Xall;
00513   R.calculate(errory, false);
00514 }
00515 */
00516 
00517 
00518 // synthesize video
00519 Mat  Dytex::synthesize(RNG &rng, int tau, bool video_output, bool display_video, double fps, double alpha, double beta) {
00520 
00521   // TODO use scaling
00522 
00523   // initialize model
00524   int t = 0;
00525   Mat Xtc = Mat(dtopt.n, 1, OPT_MAT_TYPE);  // current state
00526   Mat Xtp = Mat(dtopt.n, 1, OPT_MAT_TYPE);  // previous state
00527   Mat rvn = Mat(dtopt.n, 1, OPT_MAT_TYPE);   // random vector
00528   Mat Ytc = Mat(dtopt.m, 1, OPT_MAT_TYPE);  // current image vector
00529 
00530   // initialize output video
00531   Mat Yout;  
00532   Mat u8Y;
00533   if (!display_video) {
00534     CV_Assert(tau>=0);
00535     if (video_output) {
00536       CV_Assert(vrows != 0 && vcols != 0);
00537       Yout = MatVid::create(tau, vrows, vcols, OPT_MAT_TYPE);
00538     } else {
00539       Yout = Mat(dtopt.m, tau, OPT_MAT_TYPE);
00540     }
00541   } else {
00542     CV_Assert(vrows != 0 && vcols != 0);
00543     namedWindow("Dytex::synthesize", CV_WINDOW_AUTOSIZE);
00544     cvMoveWindow("Dytex::synthesize", 100, 100);
00545     u8Y = Mat(vrows, vcols, CV_8UC1);
00546   }
00547 
00548   // run the model for tau frames, (or forever when tau<0)
00549   while( (tau < 0) || (t < tau) ) {
00550     // calculate state vector
00551     if (t == 0) {
00552       // initial state: N(mu0, S0)
00553       mu0.copyTo(Xtc);
00554       S0.randvec(rvn, rng);
00555       Xtc += rvn;
00556 
00557     } else {
00558       // state update: Xtc = Ahat*Xtp + N(0,Q)
00559       Q.randvec(rvn, rng);
00560       Xtc = A*Xtp + rvn;
00561     }
00562     
00563     // calculate image vector
00564     Ytc = C*Xtc;
00565     if (dtopt.Yopt == DytexOptions::NONZERO_YMEAN)
00566       Ytc += Ymean;
00567 
00568     //cout << "** t=" << t << " **\n";
00569     //cout << "Xt=";
00570     //dumpMatSize(Xtc);
00571     //cout << Xtc << "\n";
00572     //cout << "Yt=";
00573     //dumpMatSize(Ytc);
00574     //cout << Ytc << "\n";
00575     
00576     // display image
00577     if (display_video) {   
00578       // convert to 8-bit w/ scaling
00579       Ytc.convertTo(u8Y, CV_8UC1, alpha, beta);
00580 
00581       Mat cframe = MatVid::unvectorize1(u8Y, vrows, vcols);
00582 
00583       //cout << cframe << "\n";
00584 
00585       imshow("Dytex::synthesize", cframe);
00586       waitKey((int)(1000.0/fps));
00587       //waitKey(0);
00588     } else {
00589 
00590       // save output
00591       if (video_output) {
00592         // save as frame
00593         Mat fY  = MatVid::unvectorize1(Ytc, vrows, vcols);
00594         Mat myf = MatVid::frame(Yout, t);
00595         fY.copyTo(myf);
00596 
00597       } else {
00598         // save as column vector
00599         Mat Youtc = Yout.col(t);
00600         Ytc.copyTo(Youtc);
00601       }
00602     }
00603   
00604     // update time step
00605     Xtc.copyTo(Xtp);
00606     t++;
00607   } // end while
00608 
00609   return Yout;
00610 }
00611   
00612 
00613 
00614 // debugging
00615 
00616 // convert to string (handles all child classes!)
00617 ostream& operator<<(ostream& output, const Dytex& dt) {
00618   // call virtual dump function.
00619   dt.dump(output);
00620   return output;
00621 }
00622 
00623 void Dytex::dump_matlab(ostream &output) const {
00624   output << "clear dt;\n";
00625   output << "dt.Ymean=" << Ymean << ";\n";
00626   output << "dt.Ahat="  << A << ";\n";
00627   output << "dt.Chat="  << C << ";\n";
00628   output << "dt.mu0="   << mu0 << ";\n";
00629   output << "dt.Rhat="  << R.mtx << ";\n";
00630   output << "dt.Qhat="  << Q.mtx << ";\n";
00631   output << "dt.S0="    << S0.mtx << ";\n";  
00632 }
00633 
00634 void Dytex::dump(ostream& output) const {
00635   output << "Dytex ({" << dtopt << "}; ";
00636   output << "Ymean[" << Ymean.rows << "x" << Ymean.cols << "]; ";
00637   output << "A[" << A.rows << "x" << A.cols << "]; ";
00638   output << "C[" << C.rows << "x" << C.cols << "]; ";
00639   output << "mu0[" << mu0.rows << "x" << mu0.cols << "]; ";
00640   output << "R[" << R << "]; ";
00641   output << "Q[" << Q << "]; ";
00642   output << "S0[" << S0 << "]";
00643   output << ")";
00644 }
00645 
00646 
00647 
00648 
00649 

Generated on 18 Dec 2014 for libdt-OpenCV by  doxygen 1.6.1