TLD(Tracking-Learning-Detection)算法学习与源码解析（三）之 tld.cpp源码解析

         本序列文章的目的是总结一下这段时间所学到的，主要分为以下几部分，本章是第三部分。

1 算法概述
2 runtld.cpp源码解析
3 tld.cpp源码解析
4 LKTracker(重点)
5 FerNNClassifier.cpp源码解析（重点）
6 tld_utils.cpp源码解析

tld的算法在tld.cpp这个文件体现，其中用到了跟踪模块和分类器，后两者在另外的文件实现。

这个文件，围绕下面这两个函数看，其他函数都是被这两个函数调用。

TLD::init这个函数是处理第一帧的图像

TLD::processFrame是处理以后每一帧图像

看懂了init再看processFrame。

/*
* TLD.cpp
*
*  Created on: Jun 9, 2011
*      Author: alantrrs
*/

#include <TLD.h>
#include <stdio.h>
using namespace cv;
using namespace std;

TLD::TLD()
{
}
TLD::TLD(const FileNode& file){
read(file);
}
/**
*初始化相关参数
*/
void TLD::read(const FileNode& file){
///Bounding Box Parameters
min_win = (int)file["min_win"];
///Genarator Parameters
//initial parameters for positive examples
patch_size = (int)file["patch_size"];
num_closest_init = (int)file["num_closest_init"];
num_warps_init = (int)file["num_warps_init"];
noise_init = (int)file["noise_init"];
angle_init = (float)file["angle_init"];
shift_init = (float)file["shift_init"];
scale_init = (float)file["scale_init"];
//update parameters for positive examples
num_closest_update = (int)file["num_closest_update"];
num_warps_update = (int)file["num_warps_update"];
noise_update = (int)file["noise_update"];
angle_update = (float)file["angle_update"];
shift_update = (float)file["shift_update"];
scale_update = (float)file["scale_update"];
//parameters for negative examples
bad_overlap = (float)file["overlap"];
bad_patches = (int)file["num_patches"];
classifier.read(file);
}
/**
*根据第一帧图像来初始化TLD
*处理以后的图像跟第一帧差不多，只不过第一帧少了纠正的过程
*/
void TLD::init(const Mat& frame1,const Rect& box,FILE* bb_file){
//bb_file = fopen("bounding_boxes.txt","w");
//Get Bounding Boxes
buildGrid(frame1,box);//在图片frame1算出所有的扫描窗口，并计算出这些扫描窗口与box的重叠度。
//当然还将frame1缩放成不同尺寸
printf("Created %d bounding boxes\n",(int)grid.size());
///Preparation
//allocation
iisum.create(frame1.rows+1,frame1.cols+1,CV_32F);
iisqsum.create(frame1.rows+1,frame1.cols+1,CV_64F);
dconf.reserve(100);
dbb.reserve(100);
bbox_step =7;
//tmp.conf.reserve(grid.size());
tmp.conf = vector<float>(grid.size());
tmp.patt = vector<vector<int> >(grid.size(),vector<int>(10,0));
//tmp.patt.reserve(grid.size());
dt.bb.reserve(grid.size());
good_boxes.reserve(grid.size());
bad_boxes.reserve(grid.size());
pEx.create(patch_size,patch_size,CV_64F);
//Init Generator
generator = PatchGenerator (0,0,noise_init,true,1-scale_init,1+scale_init,-angle_init*CV_PI/180,angle_init*CV_PI/180,-angle_init*CV_PI/180,angle_init*CV_PI/180);
getOverlappingBoxes(box,num_closest_init);//根据重叠读度找出好的窗口和坏的窗口
printf("Found %d good boxes, %d bad boxes\n",(int)good_boxes.size(),(int)bad_boxes.size());
printf("Best Box: %d %d %d %d\n",best_box.x,best_box.y,best_box.width,best_box.height);
printf("Bounding box hull: %d %d %d %d\n",bbhull.x,bbhull.y,bbhull.width,bbhull.height);
//Correct Bounding Box
lastbox=best_box;
lastconf=1;
lastvalid=true;
//Print
fprintf(bb_file,"%d,%d,%d,%d,%f\n",lastbox.x,lastbox.y,lastbox.br().x,lastbox.br().y,lastconf);
//Prepare Classifier
classifier.prepare(scales);//prepare这个函数主要是初始化集合分类器模块
///Generate Data
// Generate positive data
generatePositiveData(frame1,num_warps_init);//产生正样本fern特征和nn特征
//每个good_box都进行20次这种几何变换，那么10个box将产生200个仿射变换的bounding box，作为正样本
// Set variance threshold
Scalar stdev, mean;
meanStdDev(frame1(best_box),mean,stdev);
integral(frame1,iisum,iisqsum);//计算积分图，为了以后算方差使用
var = pow(stdev.val[0],2)*0.5; //getVar(best_box,iisum,iisqsum);
cout << "variance: " << var << endl;
//check variance
double vr =  getVar(best_box,iisum,iisqsum)*0.5;
cout << "check variance: " << vr << endl;
// Generate negative data
generateNegativeData(frame1);//产生负样本fern特征和nn特征
//Split Negative Ferns into Training and Testing sets (they are already shuffled)把负样本的fern特征分为训练集和测试集
int half = (int)nX.size()*0.5f;
nXT.assign(nX.begin()+half,nX.end());
nX.resize(half);
///Split Negative NN Examples into Training and Testing sets把负样本的nn特征分为训练集和测试集
half = (int)nEx.size()*0.5f;
nExT.assign(nEx.begin()+half,nEx.end());
nEx.resize(half);
//Merge Negative Data with Positive Data and shuffle it把正负样本的数据放到一起
vector<pair<vector<int>,int> > ferns_data(nX.size()+pX.size());
vector<int> idx = index_shuffle(0,ferns_data.size());
int a=0;
for (int i=0;i<pX.size();i++){
ferns_data[idx[a]] = pX[i];
a++;
}
for (int i=0;i<nX.size();i++){
ferns_data[idx[a]] = nX[i];
a++;
}
//Data already have been shuffled, just putting it in the same vector
vector<cv::Mat> nn_data(nEx.size()+1);
nn_data[0] = pEx;
for (int i=0;i<nEx.size();i++){
nn_data[i+1]= nEx[i];
}
///Training
classifier.trainF(ferns_data,2); //bootstrap = 2  训练 fern分类器（森林）
classifier.trainNN(nn_data);//训练  最近邻（nnc）分类器
///Threshold Evaluation on testing sets
classifier.evaluateTh(nXT,nExT);//重新计算阈值
}

/* Generate Positive data产生正样本的fern特征和nn特征
* Inputs:
* - good_boxes (bbP)
* - best_box (bbP0)
* - frame (im0)
* Outputs:
* - Positive fern features (pX)
* - Positive NN examples (pEx)
*/
void TLD::generatePositiveData(const Mat& frame, int num_warps){
Scalar mean;
Scalar stdev;
getPattern(frame(best_box),pEx,mean,stdev);//把均值化好的放到pEx,其实nn分类器的样本就这么一个
//frame(best_box)看意思是从整个frame找出best_box对应的patch
//原来box的类继承了 Rect

//Get Fern features on warped patches
Mat img;
Mat warped;
GaussianBlur(frame,img,Size(9,9),1.5);//高斯滤波保存到img
warped = img(bbhull);//算出包裹所有good_box对应的patch
RNG& rng = theRNG();
Point2f pt(bbhull.x+(bbhull.width-1)*0.5f,bbhull.y+(bbhull.height-1)*0.5f);
vector<int> fern(classifier.getNumStructs());
pX.clear();
Mat patch;
if (pX.capacity()<num_warps*good_boxes.size())
pX.reserve(num_warps*good_boxes.size());//限制正fern特征的数量
int idx;
for (int i=0;i<num_warps;i++){//总觉得这段代码有问题，img是大图，warped小图
if (i>0)
generator(frame,pt,warped,bbhull.size(),rng);//对图像区域进行仿射变换
for (int b=0;b<good_boxes.size();b++){
idx=good_boxes[b];
patch = img(grid[idx]);//warped引用对img的局部引用,warped变了，img也变
classifier.getFeatures(patch,grid[idx].sidx,fern);//函数得到输入的patch的特征fern（13位的二进制代码）
pX.push_back(make_pair(fern,1));
}
}
printf("Positive examples generated: ferns:%d NN:1\n",(int)pX.size());
}

/**
* 算出期望mean和标准差，把img均值化为0，缩放为15*15
*/
void TLD::getPattern(const Mat& img, Mat& pattern,Scalar& mean,Scalar& stdev){
//Output: resized Zero-Mean patch
resize(img,pattern,Size(patch_size,patch_size));
meanStdDev(pattern,mean,stdev);
pattern.convertTo(pattern,CV_32F);
pattern = pattern-mean.val[0];
}
/* 产生负样本的fern特征和nn特征
* Inputs:
* - Image
* - bad_boxes (Boxes far from the bounding box)
* - variance (pEx variance)
* Outputs
* - Negative fern features (nX)
* - Negative NN examples (nEx)
*/
void TLD::generateNegativeData(const Mat& frame){
random_shuffle(bad_boxes.begin(),bad_boxes.end());//Random shuffle bad_boxes indexes
int idx;
//Get Fern Features of the boxes with big variance (calculated using integral images)
int a=0;
//int num = std::min((int)bad_boxes.size(),(int)bad_patches*100); //limits the size of bad_boxes to try
printf("negative data generation started.\n");
vector<int> fern(classifier.getNumStructs());
nX.reserve(bad_boxes.size());
Mat patch;
for (int j=0;j<bad_boxes.size();j++){
idx = bad_boxes[j];
if (getVar(grid[idx],iisum,iisqsum)<var*0.5f)//去掉方差较小的负样本
continue;
patch =  frame(grid[idx]);
classifier.getFeatures(patch,grid[idx].sidx,fern);
nX.push_back(make_pair(fern,0));
a++;
}
printf("Negative examples generated: ferns: %d ",a);
//random_shuffle(bad_boxes.begin(),bad_boxes.begin()+bad_patches);//Randomly selects 'bad_patches' and get the patterns for NN;
Scalar dum1, dum2;
nEx=vector<Mat>(bad_patches);
for (int i=0;i<bad_patches;i++){
idx=bad_boxes[i];
patch = frame(grid[idx]);
getPattern(patch,nEx[i],dum1,dum2);
}
printf("NN: %d\n",(int)nEx.size());
}
/**
* 该函数通过积分图像计算输入的box的方差
*/
double TLD::getVar(const BoundingBox& box,const Mat& sum,const Mat& sqsum){
double brs = sum.at<int>(box.y+box.height,box.x+box.width);
double bls = sum.at<int>(box.y+box.height,box.x);
double trs = sum.at<int>(box.y,box.x+box.width);
double tls = sum.at<int>(box.y,box.x);
double brsq = sqsum.at<double>(box.y+box.height,box.x+box.width);
double blsq = sqsum.at<double>(box.y+box.height,box.x);
double trsq = sqsum.at<double>(box.y,box.x+box.width);
double tlsq = sqsum.at<double>(box.y,box.x);
double mean = (brs+tls-trs-bls)/((double)box.area());
double sqmean = (brsq+tlsq-trsq-blsq)/((double)box.area());
return sqmean-mean*mean;
}

/**
* 输出：
* points1：按照固定步长在某个box产生的网格点
* points2：由img1中points1在img2跟踪预测到的
* bbnext：预测到的box
*/
void TLD::processFrame(const cv::Mat& img1,const cv::Mat& img2,vector<Point2f>& points1,vector<Point2f>& points2,BoundingBox& bbnext,bool& lastboxfound, bool tl, FILE* bb_file){
vector<BoundingBox> cbb;
vector<float> cconf;
int confident_detections=0;
int didx; //detection index
///Track
if(lastboxfound && tl){
track(img1,img2,points1,points2);//跟踪
}
else{
tracked = false;
}
///Detect
detect(img2);//检测，可能会检测到多个目标窗口
///Integration
if (tracked){//如果跟踪到了
bbnext=tbb;
lastconf=tconf;
lastvalid=tvalid;
printf("Tracked\n");
if(detected){                                               //   if Detected
clusterConf(dbb,dconf,cbb,cconf);                       //   cluster detections
//dbb 聚类前的boxes，cbb聚类后
//通过 重叠度 对检测器检测到的目标bounding box进行聚类，每个类其重叠度小于0.5
//这个聚类算法细节还没看，先知道作用就好。(*^__^*) 嘻嘻……，肯定比我之前写的聚类算法好，到时再看
printf("Found %d clusters\n",(int)cbb.size());
for (int i=0;i<cbb.size();i++){
if (bbOverlap(tbb,cbb[i])<0.5 && cconf[i]>tconf){
//  Get index of a clusters that is far from tracker and are more confident than the tracker
//找到与跟踪器跟踪到的box距离比较远的类（检测器检测到的box），而且它的相关相似度比跟踪器的要大
confident_detections++;
didx=i; //detection index
}
}
if (confident_detections==1){
//if there is ONE such a cluster, re-initialize the tracker
//如果只有一个满足上述条件的box，那么就用这个目标box来重新初始化跟踪器（也就是用检测器的结果去纠正跟踪器）
printf("Found a better match..reinitializing tracking\n");
bbnext=cbb[didx];
lastconf=cconf[didx];
lastvalid=false;
}
else {
printf("%d confident cluster was found\n",confident_detections);
int cx=0,cy=0,cw=0,ch=0;
int close_detections=0;
for (int i=0;i<dbb.size();i++){
if(bbOverlap(tbb,dbb[i])>0.7){                     // Get mean of close detections
cx += dbb[i].x;
cy +=dbb[i].y;
cw += dbb[i].width;
ch += dbb[i].height;
close_detections++;
printf("weighted detection: %d %d %d %d\n",dbb[i].x,dbb[i].y,dbb[i].width,dbb[i].height);
}
}
if (close_detections>0){
bbnext.x = cvRound((float)(10*tbb.x+cx)/(float)(10+close_detections));   // weighted average trackers trajectory with the close detections
bbnext.y = cvRound((float)(10*tbb.y+cy)/(float)(10+close_detections));
bbnext.width = cvRound((float)(10*tbb.width+cw)/(float)(10+close_detections));
bbnext.height =  cvRound((float)(10*tbb.height+ch)/(float)(10+close_detections));
printf("Tracker bb: %d %d %d %d\n",tbb.x,tbb.y,tbb.width,tbb.height);
printf("Average bb: %d %d %d %d\n",bbnext.x,bbnext.y,bbnext.width,bbnext.height);
printf("Weighting %d close detection(s) with tracker..\n",close_detections);
}
else{
printf("%d close detections were found\n",close_detections);

}
}
}
}
else{                                       //   If NOT tracking
printf("Not tracking..\n");
lastboxfound = false;
lastvalid = false;
if(detected){                           //  and detector is defined
clusterConf(dbb,dconf,cbb,cconf);   //  cluster detections
printf("Found %d clusters\n",(int)cbb.size());
if (cconf.size()==1){
bbnext=cbb[0];
lastconf=cconf[0];
printf("Confident detection..reinitializing tracker\n");
lastboxfound = true;
}
}
}
lastbox=bbnext;
if (lastboxfound)
fprintf(bb_file,"%d,%d,%d,%d,%f\n",lastbox.x,lastbox.y,lastbox.br().x,lastbox.br().y,lastconf);
else
fprintf(bb_file,"NaN,NaN,NaN,NaN,NaN\n");
if (lastvalid && tl)
learn(img2);
}

/**
* 跟踪找出预测的box和自信度，跟踪到的点等
*/
void TLD::track(const Mat& img1, const Mat& img2,vector<Point2f>& points1,vector<Point2f>& points2){
/*Inputs:
* -current frame(img2), last frame(img1), last Bbox(bbox_f[0]).
*Outputs:
*- Confidence(tconf), Predicted bounding box(tbb),Validity(tvalid), points2 (for display purposes only)
*/
//Generate points
bbPoints(points1,lastbox);//在box中，按照固定步长产生一些点放到points1中
if (points1.size()<1){
printf("BB= %d %d %d %d, Points not generated\n",lastbox.x,lastbox.y,lastbox.width,lastbox.height);
tvalid=false;
tracked=false;
return;
}
vector<Point2f> points = points1;
//Frame-to-frame tracking with forward-backward error cheking
tracked = tracker.trackf2f(img1,img2,points,points2);//在一张图中跟踪另一张图中的一些点，根据points找出points2
if (tracked){
//Bounding box prediction
bbPredict(points,points2,lastbox,tbb);//预测到的box放到tbb存储
if (tracker.getFB()>10 || tbb.x>img2.cols ||  tbb.y>img2.rows || tbb.br().x < 1 || tbb.br().y <1){
tvalid =false; //too unstable prediction or bounding box out of image
tracked = false;
printf("Too unstable predictions FB error=%f\n",tracker.getFB());
return;
}
//Estimate Confidence and Validity
Mat pattern;
Scalar mean, stdev;
BoundingBox bb;
bb.x = max(tbb.x,0);
bb.y = max(tbb.y,0);
bb.width = min(min(img2.cols-tbb.x,tbb.width),min(tbb.width,tbb.br().x));
bb.height = min(min(img2.rows-tbb.y,tbb.height),min(tbb.height,tbb.br().y));
getPattern(img2(bb),pattern,mean,stdev);
vector<int> isin;
float dummy;
classifier.NNConf(pattern,isin,dummy,tconf); //Conservative Similarity
tvalid = lastvalid;
if (tconf>classifier.thr_nn_valid){
tvalid =true;
}
}
else
printf("No points tracked\n");

}

/**
* 在box中，按照固定步长产生一些点放到points中
* 最多10*10个？
*/
void TLD::bbPoints(vector<cv::Point2f>& points,const BoundingBox& bb){
int max_pts=10;
int margin_h=0;
int margin_v=0;
int stepx = ceil((bb.width-2*margin_h)/max_pts);
int stepy = ceil((bb.height-2*margin_v)/max_pts);
for (int y=bb.y+margin_v;y<bb.y+bb.height-margin_v;y+=stepy){
for (int x=bb.x+margin_h;x<bb.x+bb.width-margin_h;x+=stepx){
points.push_back(Point2f(x,y));
}
}
}

//预测在某个box中
void TLD::bbPredict(const vector<cv::Point2f>& points1,const vector<cv::Point2f>& points2,
const BoundingBox& bb1,BoundingBox& bb2)    {
int npoints = (int)points1.size();
vector<float> xoff(npoints);
vector<float> yoff(npoints);
printf("tracked points : %d\n",npoints);
for (int i=0;i<npoints;i++){
xoff[i]=points2[i].x-points1[i].x;
yoff[i]=points2[i].y-points1[i].y;
}
float dx = median(xoff);
float dy = median(yoff);
float s;
if (npoints>1){
vector<float> d;
d.reserve(npoints*(npoints-1)/2);
for (int i=0;i<npoints;i++){
for (int j=i+1;j<npoints;j++){
d.push_back(norm(points2[i]-points2[j])/norm(points1[i]-points1[j]));
}
}
s = median(d);
}
else {
s = 1.0;
}
float s1 = 0.5*(s-1)*bb1.width;
float s2 = 0.5*(s-1)*bb1.height;
printf("s= %f s1= %f s2= %f \n",s,s1,s2);
bb2.x = round( bb1.x + dx -s1);
bb2.y = round( bb1.y + dy -s2);
bb2.width = round(bb1.width*s);
bb2.height = round(bb1.height*s);
printf("predicted bb: %d %d %d %d\n",bb2.x,bb2.y,bb2.br().x,bb2.br().y);
}
/**
* 扫描所有窗口，通过分类器进行预测，可能会找出多个目标窗口
*/
void TLD::detect(const cv::Mat& frame){
//cleaning
dbb.clear();
dconf.clear();
dt.bb.clear();
double t = (double)getTickCount();
Mat img(frame.rows,frame.cols,CV_8U);
integral(frame,iisum,iisqsum);
GaussianBlur(frame,img,Size(9,9),1.5);
int numtrees = classifier.getNumStructs();
float fern_th = classifier.getFernTh();
vector <int> ferns(10);
float conf;
int a=0;
Mat patch;
for (int i=0;i<grid.size();i++){//FIXME: BottleNeck
if (getVar(grid[i],iisum,iisqsum)>=var){
a++;
patch = img(grid[i]);
classifier.getFeatures(patch,grid[i].sidx,ferns);
conf = classifier.measure_forest(ferns);
tmp.conf[i]=conf;
tmp.patt[i]=ferns;
if (conf>numtrees*fern_th){
dt.bb.push_back(i);
}
}
else
tmp.conf[i]=0.0;
}
int detections = dt.bb.size();
printf("%d Bounding boxes passed the variance filter\n",a);
printf("%d Initial detection from Fern Classifier\n",detections);
if (detections>100){
nth_element(dt.bb.begin(),dt.bb.begin()+100,dt.bb.end(),CComparator(tmp.conf));
dt.bb.resize(100);
detections=100;
}
//  for (int i=0;i<detections;i++){
//        drawBox(img,grid[dt.bb[i]]);
//    }
//  imshow("detections",img);
if (detections==0){
detected=false;
return;
}
printf("Fern detector made %d detections ",detections);
t=(double)getTickCount()-t;
printf("in %gms\n", t*1000/getTickFrequency());
//  Initialize detection structure
dt.patt = vector<vector<int> >(detections,vector<int>(10,0));        //  Corresponding codes of the Ensemble Classifier
dt.conf1 = vector<float>(detections);                                //  Relative Similarity (for final nearest neighbour classifier)
dt.conf2 =vector<float>(detections);                                 //  Conservative Similarity (for integration with tracker)
dt.isin = vector<vector<int> >(detections,vector<int>(3,-1));        //  Detected (isin=1) or rejected (isin=0) by nearest neighbour classifier
dt.patch = vector<Mat>(detections,Mat(patch_size,patch_size,CV_32F));//  Corresponding patches
int idx;
Scalar mean, stdev;
float nn_th = classifier.getNNTh();
for (int i=0;i<detections;i++){                                         //  for every remaining detection
idx=dt.bb[i];                                                       //  Get the detected bounding box index
patch = frame(grid[idx]);
getPattern(patch,dt.patch[i],mean,stdev);                //  Get pattern within bounding box
classifier.NNConf(dt.patch[i],dt.isin[i],dt.conf1[i],dt.conf2[i]);  //  Evaluate nearest neighbour classifier
dt.patt[i]=tmp.patt[idx];
//printf("Testing feature %d, conf:%f isin:(%d|%d|%d)\n",i,dt.conf1[i],dt.isin[i][0],dt.isin[i][1],dt.isin[i][2]);
if (dt.conf1[i]>nn_th){                                               //  idx = dt.conf1 > tld.model.thr_nn; % get all indexes that made it through the nearest neighbour
dbb.push_back(grid[idx]);                                         //  BB    = dt.bb(:,idx); % bounding boxes
dconf.push_back(dt.conf2[i]);                                     //  Conf  = dt.conf2(:,idx); % conservative confidences
}
}                                                                         //  end
if (dbb.size()>0){
printf("Found %d NN matches\n",(int)dbb.size());
detected=true;
}
else{
printf("No NN matches found.\n");
detected=false;
}
}

void TLD::evaluate(){
}

/**
* 根据跟踪模块的结果来优化检测模块
*/
void TLD::learn(const Mat& img){
printf("[Learning] ");
///Check consistency
BoundingBox bb;
bb.x = max(lastbox.x,0);
bb.y = max(lastbox.y,0);
bb.width = min(min(img.cols-lastbox.x,lastbox.width),min(lastbox.width,lastbox.br().x));
bb.height = min(min(img.rows-lastbox.y,lastbox.height),min(lastbox.height,lastbox.br().y));
Scalar mean, stdev;
Mat pattern;
getPattern(img(bb),pattern,mean,stdev);//将图片均值化为0，缩放为15*15
vector<int> isin;
float dummy, conf;
classifier.NNConf(pattern,isin,conf,dummy);//计算相似度
if (conf<0.5) {
printf("Fast change..not training\n");
lastvalid =false;
return;
}
if (pow(stdev.val[0],2)<var){
printf("Low variance..not training\n");
lastvalid=false;
return;
}
if(isin[2]==1){
printf("Patch in negative data..not traing");
lastvalid=false;
return;
}
/// Data generation
for (int i=0;i<grid.size();i++){
grid[i].overlap = bbOverlap(lastbox,grid[i]);//计算跟目标窗口的重叠度
}
vector<pair<vector<int>,int> > fern_examples;
good_boxes.clear();
bad_boxes.clear();
getOverlappingBoxes(lastbox,num_closest_update);//找出跟目标窗口比较近的box
if (good_boxes.size()>0)
generatePositiveData(img,num_warps_update);//产生正样本数据
else{
lastvalid = false;
printf("No good boxes..Not training");
return;
}
fern_examples.reserve(pX.size()+bad_boxes.size());
fern_examples.assign(pX.begin(),pX.end());
int idx;
for (int i=0;i<bad_boxes.size();i++){
idx=bad_boxes[i];
if (tmp.conf[idx]>=1){
fern_examples.push_back(make_pair(tmp.patt[idx],0));
}
}
vector<Mat> nn_examples;
nn_examples.reserve(dt.bb.size()+1);
nn_examples.push_back(pEx);
for (int i=0;i<dt.bb.size();i++){
idx = dt.bb[i];
if (bbOverlap(lastbox,grid[idx]) < bad_overlap)
nn_examples.push_back(dt.patch[i]);
}
/// Classifiers update
classifier.trainF(fern_examples,2);
classifier.trainNN(nn_examples);
classifier.show();
}
/**
* 在图片img算出所有的扫描窗口，并计算出这些扫描窗口与box的重叠度。
*/
void TLD::buildGrid(const cv::Mat& img, const cv::Rect& box){
const float SHIFT = 0.1;
const float SCALES[] = {0.16151,0.19381,0.23257,0.27908,0.33490,0.40188,0.48225,
0.57870,0.69444,0.83333,1,1.20000,1.44000,1.72800,
2.07360,2.48832,2.98598,3.58318,4.29982,5.15978,6.19174};//缩放成21种不同的size的img
int width, height, min_bb_side;
//Rect bbox;
BoundingBox bbox;
Size scale;
int sc=0;
for (int s=0;s<21;s++){
width = round(box.width*SCALES[s]);
height = round(box.height*SCALES[s]);
min_bb_side = min(height,width);
if (min_bb_side < min_win || width > img.cols || height > img.rows)
continue;
scale.width = width;
scale.height = height;
scales.push_back(scale);
for (int y=1;y<img.rows-height;y+=round(SHIFT*min_bb_side)){
for (int x=1;x<img.cols-width;x+=round(SHIFT*min_bb_side)){
bbox.x = x;
bbox.y = y;
bbox.width = width;
bbox.height = height;
bbox.overlap = bbOverlap(bbox,BoundingBox(box));
bbox.sidx = sc;
grid.push_back(bbox);
}
}
sc++;
}
}
/**
* 根据物理位置算出两个box的重叠度
*/
float TLD::bbOverlap(const BoundingBox& box1,const BoundingBox& box2){
if (box1.x > box2.x+box2.width) { return 0.0; }
if (box1.y > box2.y+box2.height) { return 0.0; }
if (box1.x+box1.width < box2.x) { return 0.0; }
if (box1.y+box1.height < box2.y) { return 0.0; }

float colInt =  min(box1.x+box1.width,box2.x+box2.width) - max(box1.x, box2.x);
float rowInt =  min(box1.y+box1.height,box2.y+box2.height) - max(box1.y,box2.y);

float intersection = colInt * rowInt;
float area1 = box1.width*box1.height;
float area2 = box2.width*box2.height;
return intersection / (area1 + area2 - intersection);
}

/**
* 根据max_overlap把box分类为best,good,bad
* good只保留最优的num_closest个
*/
void TLD::getOverlappingBoxes(const cv::Rect& box1,int num_closest){
float max_overlap = 0;
for (int i=0;i<grid.size();i++){
if (grid[i].overlap > max_overlap) {
max_overlap = grid[i].overlap;
best_box = grid[i];
}
if (grid[i].overlap > 0.6){
good_boxes.push_back(i);
}
else if (grid[i].overlap < bad_overlap){
bad_boxes.push_back(i);
}
}
//Get the best num_closest (10) boxes and puts them in good_boxes
if (good_boxes.size()>num_closest){
std::nth_element(good_boxes.begin(),good_boxes.begin()+num_closest,good_boxes.end(),OComparator(grid));
good_boxes.resize(num_closest);
}
getBBHull();
}
/**
* 找出包含所有good_boxes的最小的矩形
*/
void TLD::getBBHull(){
int x1=INT_MAX, x2=0;
int y1=INT_MAX, y2=0;
int idx;
for (int i=0;i<good_boxes.size();i++){
idx= good_boxes[i];
x1=min(grid[idx].x,x1);
y1=min(grid[idx].y,y1);
x2=max(grid[idx].x+grid[idx].width,x2);
y2=max(grid[idx].y+grid[idx].height,y2);
}
bbhull.x = x1;
bbhull.y = y1;
bbhull.width = x2-x1;
bbhull.height = y2 -y1;
}

bool bbcomp(const BoundingBox& b1,const BoundingBox& b2){
TLD t;
if (t.bbOverlap(b1,b2)<0.5)
return false;
else
return true;
}
int TLD::clusterBB(const vector<BoundingBox>& dbb,vector<int>& indexes){
//FIXME: Conditional jump or move depends on uninitialised value(s)
const int c = dbb.size();
//1. Build proximity matrix
Mat D(c,c,CV_32F);
float d;
for (int i=0;i<c;i++){
for (int j=i+1;j<c;j++){
d = 1-bbOverlap(dbb[i],dbb[j]);
D.at<float>(i,j) = d;
D.at<float>(j,i) = d;
}
}
//2. Initialize disjoint clustering
float L[c-1]; //Level
int nodes[c-1][2];
int belongs[c];
int m=c;
for (int i=0;i<c;i++){
belongs[i]=i;
}
for (int it=0;it<c-1;it++){
//3. Find nearest neighbor
float min_d = 1;
int node_a, node_b;
for (int i=0;i<D.rows;i++){
for (int j=i+1;j<D.cols;j++){
if (D.at<float>(i,j)<min_d && belongs[i]!=belongs[j]){
min_d = D.at<float>(i,j);
node_a = i;
node_b = j;
}
}
}
if (min_d>0.5){
int max_idx =0;
bool visited;
for (int j=0;j<c;j++){
visited = false;
for(int i=0;i<2*c-1;i++){
if (belongs[j]==i){
indexes[j]=max_idx;
visited = true;
}
}
if (visited)
max_idx++;
}
return max_idx;
}

//4. Merge clusters and assign level
L[m]=min_d;
nodes[it][0] = belongs[node_a];
nodes[it][1] = belongs[node_b];
for (int k=0;k<c;k++){
if (belongs[k]==belongs[node_a] || belongs[k]==belongs[node_b])
belongs[k]=m;
}
m++;
}
return 1;

}
/**
* dbb聚类前的boxes
* cbb聚类后的boxes
*/
void TLD::clusterConf(const vector<BoundingBox>& dbb,const vector<float>& dconf,vector<BoundingBox>& cbb,vector<float>& cconf){
int numbb =dbb.size();
vector<int> T;
float space_thr = 0.5;
int c=1;
switch (numbb){
case 1:
cbb=vector<BoundingBox>(1,dbb[0]);
cconf=vector<float>(1,dconf[0]);
return;
break;
case 2:
T =vector<int>(2,0);
if (1-bbOverlap(dbb[0],dbb[1])>space_thr){
T[1]=1;
c=2;
}
break;
default:
T = vector<int>(numbb,0);
c = partition(dbb,T,(*bbcomp));
//c = clusterBB(dbb,T);
break;
}
cconf=vector<float>(c);
cbb=vector<BoundingBox>(c);
printf("Cluster indexes: ");
BoundingBox bx;
for (int i=0;i<c;i++){
float cnf=0;
int N=0,mx=0,my=0,mw=0,mh=0;
for (int j=0;j<T.size();j++){
if (T[j]==i){
printf("%d ",i);
cnf=cnf+dconf[j];
mx=mx+dbb[j].x;
my=my+dbb[j].y;
mw=mw+dbb[j].width;
mh=mh+dbb[j].height;
N++;
}
}
if (N>0){
cconf[i]=cnf/N;
bx.x=cvRound(mx/N);
bx.y=cvRound(my/N);
bx.width=cvRound(mw/N);
bx.height=cvRound(mh/N);
cbb[i]=bx;
}
}
printf("\n");
}

注：

原作者是用matlab实现的，我分析的源码是其他大神用c++和opencv实现的，源码可以从
https://github.com/arthurv/OpenTLD或者https://github.com/alantrrs/OpenTLD下载
本序列参考了zouxy09同学的序列文章，在此表示感谢
http://blog.csdn.net/zouxy09/article/details/7893011