#include <getopt.h>
#include <libgen.h>
#include <unistd.h>
+ #include <iomanip>
#include "kcf.h"
#include "vot.hpp"
double calcAccuracy(std::string line, cv::Rect bb_rect, cv::Rect &groundtruth_rect)
{
std::vector<float> numbers;
- std::istringstream s( line );
+ std::istringstream s(line);
float x;
char ch;
- while (s >> x){
+ while (s >> x) {
numbers.push_back(x);
s >> ch;
}
double y1 = std::min(numbers[1], std::min(numbers[3], std::min(numbers[5], numbers[7])));
double y2 = std::max(numbers[1], std::max(numbers[3], std::max(numbers[5], numbers[7])));
- groundtruth_rect = cv::Rect(x1, y1, x2-x1, y2-y1);
+ groundtruth_rect = cv::Rect(x1, y1, x2 - x1, y2 - y1);
double rects_intersection = (groundtruth_rect & bb_rect).area();
double rects_union = (groundtruth_rect | bb_rect).area();
- double accuracy = rects_intersection/rects_union;
+ double accuracy = rects_intersection / rects_union;
return accuracy;
}
int main(int argc, char *argv[])
{
- //load region, images and prepare for output
+ // load region, images and prepare for output
std::string region, images, output;
int visualize_delay = -1, fit_size_x = -1, fit_size_y = -1;
KCF_Tracker tracker;
while (1) {
int option_index = 0;
- static struct option long_options[] = {
- {"debug", no_argument, 0, 'd' },
- {"help", no_argument, 0, 'h' },
- {"output", required_argument, 0, 'o' },
- {"visualize", optional_argument, 0, 'v' },
- {"fit", optional_argument, 0, 'f' },
- {0, 0, 0, 0 }
- };
-
- int c = getopt_long(argc, argv, "dhv::f::o:",
- long_options, &option_index);
- if (c == -1)
- break;
+ static struct option long_options[] = {{"debug", no_argument, 0, 'd'},
+ {"visualDebug", no_argument, 0, 'p'},
+ {"help", no_argument, 0, 'h'},
+ {"output", required_argument, 0, 'o'},
+ {"visualize", optional_argument, 0, 'v'},
+ {"fit", optional_argument, 0, 'f'},
+ {0, 0, 0, 0}};
+
+ int c = getopt_long(argc, argv, "dphv::f::o:", long_options, &option_index);
+ if (c == -1) break;
switch (c) {
case 'd':
tracker.m_debug = true;
break;
+ case 'p':
+ tracker.m_visual_debug = true;
+ visualize_delay = 500;
+ break;
case 'h':
- std::cerr << "Usage: \n"
- << argv[0] << " [options]\n"
- << argv[0] << " [options] <directory>\n"
- << argv[0] << " [options] <path/to/region.txt or groundtruth.txt> <path/to/images.txt> [path/to/output.txt]\n"
- << "Options:\n"
- << " --visualize | -v[delay_ms]\n"
- << " --output | -o <output.txt>\n"
- << " --debug | -d\n"
- << " --fit | -f[WxH]\n";
+ std::cerr
+ << "Usage: \n"
+ << argv[0] << " [options]\n"
+ << argv[0] << " [options] <directory>\n"
+ << argv[0]
+ << " [options] <path/to/region.txt or groundtruth.txt> <path/to/images.txt> [path/to/output.txt]\n"
+ << "Options:\n"
+ << " --visualize | -v[delay_ms]\n"
+ << " --output | -o <output.txt>\n"
+ << " --debug | -d\n"
+ << " --visualDebug | -p\n"
+ << " --fit | -f[WxH]\n";
exit(0);
break;
case 'o':
sizes.erase(0, pos + delimiter.length());
fit_size_x = stol(first_argument);
- fit_size_y = stol(sizes);
+ fit_size_y = stol(sizes);
break;
}
}
case 0:
region = access("groundtruth.txt", F_OK) == 0 ? "groundtruth.txt" : "region.txt";
images = "images.txt";
- if (output.empty())
- output = "output.txt";
+ if (output.empty()) output = "output.txt";
break;
case 2:
// Fall through
cv::Mat image;
- //img = firts frame, initPos = initial position in the first frame
+ // img = firts frame, initPos = initial position in the first frame
cv::Rect init_rect = vot_io.getInitRectangle();
vot_io.outputBoundingBox(init_rect);
vot_io.getNextImage(image);
cv::Rect bb_rect;
double avg_time = 0., sum_accuracy = 0.;
int frames = 0;
- while (vot_io.getNextImage(image) == 1) {
+
+ std::cout << std::fixed << std::setprecision(2);
+
+ while (vot_io.getNextImage(image) == 1){
double time_profile_counter = cv::getCPUTickCount();
tracker.track(image);
time_profile_counter = cv::getCPUTickCount() - time_profile_counter;
- std::cout << " -> speed : " << time_profile_counter/((double)cvGetTickFrequency()*1000) << "ms. per frame, "
+ std::cout << " -> speed : " << time_profile_counter/((double)cvGetTickFrequency()*1000) << "ms per frame, "
"response : " << tracker.getFilterResponse();
avg_time += time_profile_counter/((double)cvGetTickFrequency()*1000);
frames++;
bb = tracker.getBBox();
- bb_rect = cv::Rect(bb.cx - bb.w/2., bb.cy - bb.h/2., bb.w, bb.h);
+ bb_rect = cv::Rect(bb.cx - bb.w / 2., bb.cy - bb.h / 2., bb.w, bb.h);
vot_io.outputBoundingBox(bb_rect);
if (groundtruth_stream.is_open()) {
cv::Rect groundtruthRect;
double accuracy = calcAccuracy(line, bb_rect, groundtruthRect);
- if (visualize_delay >= 0)
- cv::rectangle(image, groundtruthRect, CV_RGB(255, 0,0), 1);
+ if (visualize_delay >= 0) cv::rectangle(image, groundtruthRect, CV_RGB(255, 0, 0), 1);
std::cout << ", accuracy: " << accuracy;
sum_accuracy += accuracy;
}
std::cout << std::endl;
if (visualize_delay >= 0) {
- cv::rectangle(image, bb_rect, CV_RGB(0,255,0), 2);
+ cv::Point pt(bb.cx, bb.cy);
+ cv::Size size(bb.w, bb.h);
+ cv::RotatedRect rotatedRectangle(pt, size, bb.a);
+
+ cv::Point2f vertices[4];
+ rotatedRectangle.points(vertices);
+
+ for (int i = 0; i < 4; i++)
+ cv::line(image, vertices[i], vertices[(i + 1) % 4], cv::Scalar(0, 255, 0), 2);
+ // cv::rectangle(image, cv::Rect(bb.cx - bb.w/2., bb.cy - bb.h/2., bb.w, bb.h), CV_RGB(0,255,0),
+ // 2);
+ std::string angle = std::to_string(bb.a);
+ angle.erase(angle.find_last_not_of('0') + 1, std::string::npos);
+ angle.erase(angle.find_last_not_of('.') + 1, std::string::npos);
+ cv::putText(image, "Frame: " + std::to_string(frames) + " " + angle + " angle",
+ cv::Point(0, image.rows - 1), cv::FONT_HERSHEY_SIMPLEX, 0.7, cv::Scalar(0, 255, 0), 2);
cv::imshow("output", image);
int ret = cv::waitKey(visualize_delay);
- if (visualize_delay > 0 && ret != -1 && ret != 255)
- break;
+ if (visualize_delay > 0 && ret != -1 && ret != 255) break;
}
-// std::stringstream s;
-// std::string ss;
-// int countTmp = frames;
-// s << "imgs" << "/img" << (countTmp/10000);
-// countTmp = countTmp%10000;
-// s << (countTmp/1000);
-// countTmp = countTmp%1000;
-// s << (countTmp/100);
-// countTmp = countTmp%100;
-// s << (countTmp/10);
-// countTmp = countTmp%10;
-// s << (countTmp);
-// s << ".jpg";
-// s >> ss;
-// //set image output parameters
-// std::vector<int> compression_params;
-// compression_params.push_back(CV_IMWRITE_JPEG_QUALITY);
-// compression_params.push_back(90);
-// cv::imwrite(ss.c_str(), image, compression_params);
+ // std::stringstream s;
+ // std::string ss;
+ // int countTmp = frames;
+ // s << "imgs" << "/img" << (countTmp/10000);
+ // countTmp = countTmp%10000;
+ // s << (countTmp/1000);
+ // countTmp = countTmp%1000;
+ // s << (countTmp/100);
+ // countTmp = countTmp%100;
+ // s << (countTmp/10);
+ // countTmp = countTmp%10;
+ // s << (countTmp);
+ // s << ".jpg";
+ // s >> ss;
+ // //set image output parameters
+ // std::vector<int> compression_params;
+ // compression_params.push_back(CV_IMWRITE_JPEG_QUALITY);
+ // compression_params.push_back(90);
+ // cv::imwrite(ss.c_str(), image, compression_params);
}
- std::cout << "Average processing speed: " << avg_time/frames << "ms (" << 1./(avg_time/frames)*1000 << " fps)";
+ std::cout << "Average processing speed: " << avg_time / frames << "ms (" << 1. / (avg_time / frames) * 1000 << " fps)";
if (groundtruth_stream.is_open()) {
std::cout << "; Average accuracy: " << sum_accuracy/frames << std::endl;
groundtruth_stream.close();
#endif // OPENMP
#define DEBUG_PRINT(obj) \
- if (m_debug) { \
+ if (m_debug || m_visual_debug) { \
std::cout << #obj << " @" << __LINE__ << std::endl << (obj) << std::endl; \
}
#define DEBUG_PRINTM(obj) \
<< (obj) << std::endl; \
}
+ template <typename T>
+ T clamp(const T& n, const T& lower, const T& upper)
+ {
+ return std::max(lower, std::min(n, upper));
+ }
+
+ template <typename T>
+ void clamp2(T& n, const T& lower, const T& upper)
+ {
+ n = std::max(lower, std::min(n, upper));
+ }
+
KCF_Tracker::KCF_Tracker(double padding, double kernel_sigma, double lambda, double interp_factor,
double output_sigma_factor, int cell_size)
: fft(*new FFT()), p_padding(padding), p_output_sigma_factor(output_sigma_factor), p_kernel_sigma(kernel_sigma),
p_roi.width = p_windows_size.width / p_cell_size;
p_roi.height = p_windows_size.height / p_cell_size;
- p_num_of_feats = 31;
- if (m_use_color) p_num_of_feats += 3;
- if (m_use_cnfeat) p_num_of_feats += 10;
-
p_scales.clear();
- if (m_use_scale)
+ if (m_use_scale) {
for (int i = -int(p_num_scales) / 2; i <= int(p_num_scales) / 2; ++i)
p_scales.push_back(std::pow(p_scale_step, i));
- else
+ } else {
p_scales.push_back(1.);
+ p_num_scales = 1;
+ }
+
+ if (m_use_angle) {
+ for (int i = p_angle_min; i <= p_angle_max; i += p_angle_step)
+ p_angles.push_back(i);
+ } else {
+ p_angles.push_back(0);
+ p_num_angles = 1;
+ }
#ifdef CUFFT
if (p_roi.height * (p_roi.width / 2 + 1) > 1024) {
std::cerr << "Window after forward FFT is too big for CUDA kernels. Plese use -f to set "
"the window dimensions so its size is less or equal to "
<< 1024 * p_cell_size * p_cell_size * 2 + 1
- << " pixels . Currently the size of the window is: " << p_windows_size.width << "x" << p_windows_size.height
- << " which is " << p_windows_size.width * p_windows_size.height << " pixels. " << std::endl;
+ << " pixels . Currently the size of the window is: " << p_windows_size.width << "x"
+ << p_windows_size.height << " which is " << p_windows_size.width * p_windows_size.height
+ << " pixels. " << std::endl;
std::exit(EXIT_FAILURE);
}
std::cerr << "cuFFT supports only Gaussian kernel." << std::endl;
std::exit(EXIT_FAILURE);
}
+
CudaSafeCall(cudaSetDeviceFlags(cudaDeviceMapHost));
+
p_rot_labels_data = DynMem(p_roi.width * p_roi.height * sizeof(float));
p_rot_labels = cv::Mat(p_roi, CV_32FC1, p_rot_labels_data.hostMem());
-#else
- p_xf.create(p_roi.height, p_roi.height / 2 + 1, p_num_of_feats);
#endif
#if defined(CUFFT) || defined(FFTW)
p_yf.create(p_roi.height, width, 1);
p_xf.create(p_roi.height, width, p_num_of_feats);
- int max1 = m_use_big_batch ? 2 : p_num_scales;
- int max2 = m_use_big_batch ? 1 : p_num_angles;
- int max = BIG_BATCH_MODE ? 2 : p_num_scales;
- for (int i = 0; i < max; ++i) {
- if (BIG_BATCH_MODE && i == 1)
- p_threadctxs.emplace_back(p_roi, p_num_of_feats * p_num_scales, 1, p_num_scales);
- else
- p_threadctxs.emplace_back(p_roi, p_num_of_feats, p_scales[i], 1);
++ int max1 = BIG_BATCH_MODE ? 2 : p_num_scales;
++ int max2 = BIG_BATCH_MODE ? 1 : p_num_angles;
+ for (int i = 0; i < max1; ++i) {
+ for (int j = 0; j < max2; ++j) {
- if (m_use_big_batch && i == 1)
++ if (BIG_BATCH_MODE && i == 1)
+ p_threadctxs.emplace_back(p_roi, p_num_of_feats * p_num_scales * p_num_angles, 1, 0, p_num_scales,
+ p_num_angles);
+ else
+ p_threadctxs.emplace_back(p_roi, p_num_of_feats, p_scales[i], p_angles[j]);
+ }
}
p_current_scale = 1.;
p_min_max_scale[0] = std::pow(p_scale_step, std::ceil(std::log(min_size_ratio) / log(p_scale_step)));
p_min_max_scale[1] = std::pow(p_scale_step, std::floor(std::log(max_size_ratio) / log(p_scale_step)));
- std::cout << "init: img size " << img.cols << " " << img.rows << std::endl;
- std::cout << "init: win size. " << p_windows_size.width << " " << p_windows_size.height << std::endl;
+ std::cout << "init: img size " << img.cols << "x" << img.rows << std::endl;
+ std::cout << "init: win size " << p_windows_size.width << "x" << p_windows_size.height << std::endl;
+ std::cout << "init: FFT size " << p_roi.width << "x" << p_roi.height << std::endl;
std::cout << "init: min max scales factors: " << p_min_max_scale[0] << " " << p_min_max_scale[1] << std::endl;
- p_output_sigma = std::sqrt(p_pose.w * p_pose.h) * p_output_sigma_factor / static_cast<double>(p_cell_size);
+ p_output_sigma = std::sqrt(p_pose.w * p_pose.h) * p_output_sigma_factor / p_cell_size;
- fft.init(p_roi.width, p_roi.height, p_num_of_feats, p_num_scales * p_num_angles, m_use_big_batch);
+ fft.init(p_roi.width, p_roi.height, p_num_of_feats, p_num_scales);
fft.set_window(cosine_window_function(p_roi.width, p_roi.height));
// window weights, i.e. labels
- fft.forward(
- gaussian_shaped_labels(p_output_sigma, p_roi.width, p_roi.height), p_yf,
- m_use_cuda ? p_rot_labels_data.deviceMem() : nullptr, p_threadctxs.front().stream);
+ fft.forward(gaussian_shaped_labels(p_output_sigma, p_roi.width, p_roi.height), p_yf,
+ m_use_cuda ? p_rot_labels_data.deviceMem() : nullptr, p_threadctxs.front().stream);
DEBUG_PRINTM(p_yf);
// obtain a sub-window for training initial model
- p_threadctxs.front().patch_feats.clear();
-
- std::vector<cv::Mat> patch_feats = get_features(input_rgb, input_gray, p_pose.cx, p_pose.cy,
- p_windows_size.width, p_windows_size.height);
+ int size_x_scaled = floor(p_windows_size.width);
+ int size_y_scaled = floor(p_windows_size.height);
+
+ cv::Mat patch_gray = get_subwindow(input_gray, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_gray, p_windows_size.width, p_windows_size.height, 0, false);
+
- cv::Mat patch_rgb = cv::Mat::zeros(size_y_scaled, size_x_scaled, CV_32F);
++ cv::Mat patch_rgb;
+ if ((m_use_color || m_use_cnfeat) && input_rgb.channels() == 3) {
+ patch_rgb = get_subwindow(input_rgb, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_rgb, p_windows_size.width, p_windows_size.height, 0, false);
+ }
+
- get_features(patch_rgb, patch_gray, p_threadctxs.front());
- fft.forward_window(p_threadctxs.front().patch_feats, p_model_xf, p_threadctxs.front().fw_all,
++ std::vector<cv::Mat> patch_feats = get_features(patch_rgb, patch_gray);
+ fft.forward_window(patch_feats, p_model_xf, p_threadctxs.front().fw_all,
m_use_cuda ? p_threadctxs.front().data_features.deviceMem() : nullptr,
p_threadctxs.front().stream);
DEBUG_PRINTM(p_model_xf);
+
#if !defined(BIG_BATCH) && defined(CUFFT) && (defined(ASYNC) || defined(OPENMP))
p_threadctxs.front().model_xf = p_model_xf;
p_threadctxs.front().model_xf.set_stream(p_threadctxs.front().stream);
BBox_c tmp = p_pose;
tmp.w *= p_current_scale;
tmp.h *= p_current_scale;
+ tmp.a = p_current_angle;
if (p_resize_image) tmp.scale(1 / p_downscale_factor);
if (p_fit_to_pw2) {
void KCF_Tracker::track(cv::Mat &img)
{
- if (m_debug) std::cout << "NEW FRAME" << '\n';
+ if (m_debug || m_visual_debug) std::cout << "\nNEW FRAME" << std::endl;
cv::Mat input_gray, input_rgb = img.clone();
if (img.channels() == 3) {
cv::cvtColor(img, input_gray, CV_BGR2GRAY);
cv::resize(input_rgb, input_rgb, cv::Size(0, 0), p_scale_factor_x, p_scale_factor_y, cv::INTER_LINEAR);
}
}
-
max_response = -1.;
ThreadCtx *max = nullptr;
cv::Point2i *max_response_pt = nullptr;
});
for (auto const &it : p_threadctxs)
it.async_res.wait();
-
#else // !ASYNC
- // FIXME: Iterate correctly in big batch mode - perhaps have only one element in the list
NORMAL_OMP_PARALLEL_FOR
- for (uint i = m_use_big_batch ? 1 : 0; i < p_threadctxs.size(); ++i)
- for (uint i = 0; i < p_threadctxs.size(); ++i)
++ for (uint i = BIG_BATCH_MODE ? 1 : 0; i < p_threadctxs.size(); ++i)
scale_track(p_threadctxs[i], input_rgb, input_gray);
#endif
}
}
#else
- // FIXME: Iterate correctly in big batch mode - perhaps have only one element in the list
- for (uint j = 0; j < p_scales.size(); ++j) {
- if (p_threadctxs[0].max_responses[j] > max_response) {
- max_response = p_threadctxs[0].max_responses[j];
- max_response_pt = &p_threadctxs[0].max_locs[j];
- max_response_map = &p_threadctxs[0].response_maps[j];
- max = &p_threadctxs[0];
+ for (uint j = 0; j < p_num_scales; ++j) {
+ for (uint k = 0; k < p_num_angles; ++k) {
+ if (p_threadctxs.back().max_responses[j + k] > max_response) {
+ max_response = p_threadctxs.back().max_responses[j + k];
+ max_response_pt = &p_threadctxs.back().max_locs[j + k];
+ max_response_map = &p_threadctxs.back().response_maps[j + k];
+ }
}
}
+ max = &p_threadctxs.back();
#endif
+ if (m_visual_debug) {
+ cv::Mat all_responses(cv::Size(p_num_angles* p_debug_image_size, p_num_scales * p_debug_image_size),
+ p_debug_scale_responses[0].type(), cv::Scalar::all(0));
+ cv::Mat all_subwindows(cv::Size(p_num_angles* p_debug_image_size, p_num_scales* p_debug_image_size),
+ p_debug_subwindows[0].type(), cv::Scalar::all(0));
+ for (size_t i = 0; i < p_num_scales; ++i) {
+ for (size_t j = 0; j < p_num_angles; ++j) {
+ cv::Mat in_roi(all_responses, cv::Rect(j * p_debug_image_size, i * p_debug_image_size,
+ p_debug_image_size, p_debug_image_size));
+ p_debug_scale_responses[5 * i + j].copyTo(in_roi);
+ in_roi = all_subwindows(
+ cv::Rect(j * p_debug_image_size, i * p_debug_image_size, p_debug_image_size, p_debug_image_size));
+ p_debug_subwindows[5 * i + j].copyTo(in_roi);
+ }
+ }
+ cv::namedWindow("All subwindows", CV_WINDOW_AUTOSIZE);
+ cv::imshow("All subwindows", all_subwindows);
+ cv::namedWindow("All responses", CV_WINDOW_AUTOSIZE);
+ cv::imshow("All responses", all_responses);
+ cv::waitKey();
+ p_debug_scale_responses.clear();
+ p_debug_subwindows.clear();
+ }
DEBUG_PRINTM(*max_response_map);
DEBUG_PRINT(*max_response_pt);
new_location = sub_pixel_peak(*max_response_pt, *max_response_map);
DEBUG_PRINT(new_location);
+ if (m_visual_debug) std::cout << "Old p_pose, cx: " << p_pose.cx << " cy: " << p_pose.cy << std::endl;
+
p_pose.cx += p_current_scale * p_cell_size * double(new_location.x);
p_pose.cy += p_current_scale * p_cell_size * double(new_location.y);
+
+ if (m_visual_debug) std::cout << "New p_pose, cx: " << p_pose.cx << " cy: " << p_pose.cy << std::endl;
+
if (p_fit_to_pw2) {
- if (p_pose.cx < 0) p_pose.cx = 0;
- if (p_pose.cx > (img.cols * p_scale_factor_x) - 1) p_pose.cx = (img.cols * p_scale_factor_x) - 1;
- if (p_pose.cy < 0) p_pose.cy = 0;
- if (p_pose.cy > (img.rows * p_scale_factor_y) - 1) p_pose.cy = (img.rows * p_scale_factor_y) - 1;
+ clamp2(p_pose.cx, 0.0, (img.cols * p_scale_factor_x) - 1);
+ clamp2(p_pose.cy, 0.0, (img.rows * p_scale_factor_y) - 1);
} else {
- if (p_pose.cx < 0) p_pose.cx = 0;
- if (p_pose.cx > img.cols - 1) p_pose.cx = img.cols - 1;
- if (p_pose.cy < 0) p_pose.cy = 0;
- if (p_pose.cy > img.rows - 1) p_pose.cy = img.rows - 1;
+ clamp2(p_pose.cx, 0.0, img.cols - 1.0);
+ clamp2(p_pose.cy, 0.0, img.rows - 1.0);
}
// sub grid scale interpolation
p_current_scale *= max->scale;
}
+ clamp2(p_current_scale, p_min_max_scale[0], p_min_max_scale[1]);
- ThreadCtx &ctx = p_threadctxs.front();
+ if (p_current_scale < p_min_max_scale[0]) p_current_scale = p_min_max_scale[0];
+ if (p_current_scale > p_min_max_scale[1]) p_current_scale = p_min_max_scale[1];
+
+ p_current_angle = (p_current_angle + max->angle) < 0
+ ? -std::abs(p_current_angle + max->angle) % 360
+ : (p_current_angle + max->angle) % 360;
+
// obtain a subwindow for training at newly estimated target position
- std::vector<cv::Mat> patch_feats = get_features(input_rgb, input_gray, p_pose.cx, p_pose.cy,
- p_windows_size.width, p_windows_size.height,
- p_current_scale);
- fft.forward_window(patch_feats, p_xf, ctx.fw_all,
- m_use_cuda ? ctx.data_features.deviceMem() : nullptr, ctx.stream);
+ int size_x_scaled = floor(p_windows_size.width * p_current_scale);
+ int size_y_scaled = floor(p_windows_size.height * p_current_scale);
+
+ cv::Mat patch_gray = get_subwindow(input_gray, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_gray, p_windows_size.width, p_windows_size.height, p_current_angle, false);
+
+ cv::Mat patch_rgb = cv::Mat::zeros(size_y_scaled, size_x_scaled, CV_32F);
+ if ((m_use_color || m_use_cnfeat) && input_rgb.channels() == 3) {
+ patch_rgb = get_subwindow(input_rgb, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_rgb, p_windows_size.width, p_windows_size.height, p_current_angle, false);
+ }
+
- p_threadctxs.front().patch_feats.clear();
- get_features(patch_rgb, patch_gray, p_threadctxs.front());
- fft.forward_window(p_threadctxs.front().patch_feats, p_xf, p_threadctxs.front().fw_all,
- m_use_cuda ? p_threadctxs.front().data_features.deviceMem() : nullptr, p_threadctxs.front().stream);
++ ThreadCtx &ctx = p_threadctxs.front();
++ std::vector<cv::Mat> patch_feats = get_features(patch_rgb, patch_gray);
++ fft.forward_window(patch_feats, p_xf, ctx.fw_all, m_use_cuda ? ctx.data_features.deviceMem() : nullptr, ctx.stream);
// subsequent frames, interpolate model
p_model_xf = p_model_xf * float((1. - p_interp_factor)) + p_xf * float(p_interp_factor);
alphaf_den = (p_xf * xfconj);
} else {
// Kernel Ridge Regression, calculate alphas (in Fourier domain)
- gaussian_correlation(p_threadctxs.front(), p_xf, p_xf, p_kernel_sigma,
+ gaussian_correlation(ctx, p_xf, p_xf, p_kernel_sigma,
true);
// ComplexMat alphaf = p_yf / (kf + p_lambda); //equation for fast training
// p_model_alphaf = p_model_alphaf * (1. - p_interp_factor) + alphaf * p_interp_factor;
- alphaf_num = p_yf * p_threadctxs.front().kf;
- alphaf_den = p_threadctxs.front().kf * (p_threadctxs.front().kf + float(p_lambda));
+ alphaf_num = p_yf * ctx.kf;
+ alphaf_den = ctx.kf * (ctx.kf + float(p_lambda));
}
p_model_alphaf_num = p_model_alphaf_num * float((1. - p_interp_factor)) + alphaf_num * float(p_interp_factor);
void KCF_Tracker::scale_track(ThreadCtx &vars, cv::Mat &input_rgb, cv::Mat &input_gray)
{
- if (m_use_big_batch) {
- vars.patch_feats.clear();
+ std::vector<cv::Mat> patch_feats;
+ if (BIG_BATCH_MODE) {
BIG_BATCH_OMP_PARALLEL_FOR
- for (uint i = 0; i < p_num_scales; ++i) {
- patch_feats = get_features(input_rgb, input_gray, this->p_pose.cx, this->p_pose.cy,
- this->p_windows_size.width, this->p_windows_size.height,
- this->p_current_scale * this->p_scales[i]);
+ for (uint i = 0; i < this->p_scales.size(); ++i) {
+ for (uint j = 0; j < this->p_angles.size(); ++j) {
+ int size_x_scaled = floor(this->p_windows_size.width * this->p_current_scale * this->p_scales[i]);
+ int size_y_scaled = floor(this->p_windows_size.height * this->p_current_scale * this->p_scales[i]);
+
+ cv::Mat patch_gray =
+ get_subwindow(input_gray, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_gray, p_windows_size.width, p_windows_size.height,
+ p_current_scale * this->p_scales[i], p_current_angle + this->p_angles[j]);
+
- cv::Mat patch_rgb = cv::Mat::zeros(size_y_scaled, size_x_scaled, CV_32F);
++ cv::Mat patch_rgb;
+ if ((m_use_color || m_use_cnfeat) && input_rgb.channels() == 3) {
+ patch_rgb =
+ get_subwindow(input_rgb, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_rgb, p_windows_size.width, p_windows_size.height,
+ p_current_scale * this->p_scales[i], p_current_angle + this->p_angles[j]);
+ }
- get_features(patch_rgb, patch_gray, vars);
++ std::vector<cv::Mat> tmp = get_features(patch_rgb, patch_gray);
++ BIG_BATCH_OMP_ORDERED
++ patch_feats.insert(patch_feats.end(), tmp.begin(), tmp.end());
+ }
}
} else {
- patch_feats = get_features(input_rgb, input_gray, this->p_pose.cx, this->p_pose.cy,
- this->p_windows_size.width, this->p_windows_size.height,
- this->p_current_scale * vars.scale);
+ int size_x_scaled = floor(this->p_windows_size.width * this->p_current_scale * vars.scale);
+ int size_y_scaled = floor(this->p_windows_size.height * this->p_current_scale * vars.scale);
+
+ cv::Mat patch_gray = get_subwindow(input_gray, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_gray, p_windows_size.width, p_windows_size.height, p_current_scale * vars.scale);
+
- cv::Mat patch_rgb = cv::Mat::zeros(size_y_scaled, size_x_scaled, CV_32F);
++ cv::Mat patch_rgb;
+ if ((m_use_color || m_use_cnfeat) && input_rgb.channels() == 3) {
+ patch_rgb = get_subwindow(input_rgb, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_rgb, p_windows_size.width, p_windows_size.height, p_current_scale * vars.scale,
+ p_current_angle + vars.angle);
+ }
- vars.patch_feats.clear();
- get_features(patch_rgb, patch_gray, vars);
++ patch_feats = get_features(patch_rgb, patch_gray);
}
- fft.forward_window(vars.patch_feats, vars.zf, vars.fw_all, m_use_cuda ? vars.data_features.deviceMem() : nullptr,
+ fft.forward_window(patch_feats, vars.zf, vars.fw_all, m_use_cuda ? vars.data_features.deviceMem() : nullptr,
vars.stream);
DEBUG_PRINTM(vars.zf);
if (m_use_linearkernel) {
- vars.kzf = m_use_big_batch ? (vars.zf.mul2(this->p_model_alphaf)).sum_over_channels()
+ vars.kzf = BIG_BATCH_MODE ? (vars.zf.mul2(this->p_model_alphaf)).sum_over_channels()
: (p_model_alphaf * vars.zf).sum_over_channels();
fft.inverse(vars.kzf, vars.response, m_use_cuda ? vars.data_i_1ch.deviceMem() : nullptr, vars.stream);
} else {
gaussian_correlation(vars, vars.zf, this->p_model_xf, this->p_kernel_sigma);
DEBUG_PRINTM(this->p_model_alphaf);
DEBUG_PRINTM(vars.kzf);
- vars.kzf = m_use_big_batch ? vars.kzf.mul(this->p_model_alphaf) : this->p_model_alphaf * vars.kzf;
+ vars.kzf = BIG_BATCH_MODE ? vars.kzf.mul(this->p_model_alphaf) : this->p_model_alphaf * vars.kzf;
#endif
fft.inverse(vars.kzf, vars.response, m_use_cuda ? vars.data_i_1ch.deviceMem() : nullptr, vars.stream);
}
account the fact that, if the target doesn't move, the peak
will appear at the top-left corner, not at the center (this is
discussed in the paper). the responses wrap around cyclically. */
- if (m_use_big_batch) {
- cv::split(vars.response, vars.response_maps);
-
- for (size_t i = 0; i < p_scales.size(); ++i) {
- double min_val, max_val;
- cv::Point2i min_loc, max_loc;
- cv::minMaxLoc(vars.response_maps[i], &min_val, &max_val, &min_loc, &max_loc);
- DEBUG_PRINT(max_loc);
- double weight = p_scales[i] < 1. ? p_scales[i] : 1. / p_scales[i];
- vars.max_responses[i] = max_val * weight;
- vars.max_locs[i] = max_loc;
- }
- } else {
- double min_val;
- cv::Point2i min_loc;
- cv::minMaxLoc(vars.response, &min_val, &vars.max_val, &min_loc, &vars.max_loc);
+ #ifdef BIG_BATCH
+ cv::split(vars.response, vars.response_maps);
+
+ for (size_t i = 0; i < p_scales.size(); ++i) {
+ double min_val, max_val;
+ cv::Point2i min_loc, max_loc;
+ cv::minMaxLoc(vars.response_maps[i], &min_val, &max_val, &min_loc, &max_loc);
+ DEBUG_PRINT(max_loc);
+ double weight = p_scales[i] < 1. ? p_scales[i] : 1. / p_scales[i];
+ vars.max_responses[i] = max_val * weight;
+ vars.max_locs[i] = max_loc;
+ }
+ #else
+ double min_val;
+ cv::Point2i min_loc;
+ cv::minMaxLoc(vars.response, &min_val, &vars.max_val, &min_loc, &vars.max_loc);
- DEBUG_PRINT(vars.max_loc);
+ DEBUG_PRINT(vars.max_loc);
- double weight = vars.scale < 1. ? vars.scale : 1. / vars.scale;
- vars.max_response = vars.max_val * weight;
- }
+ double weight = vars.scale < 1. ? vars.scale : 1. / vars.scale;
+ vars.max_response = vars.max_val * weight;
+ #endif
return;
}
// ****************************************************************************
- void KCF_Tracker::get_features(cv::Mat &patch_rgb, cv::Mat &patch_gray, ThreadCtx &vars)
-std::vector<cv::Mat> KCF_Tracker::get_features(cv::Mat & input_rgb, cv::Mat & input_gray, int cx, int cy, int size_x, int size_y, double scale)
++std::vector<cv::Mat> KCF_Tracker::get_features(cv::Mat &patch_rgb, cv::Mat &patch_gray)
{
- int size_x_scaled = floor(size_x * scale);
- int size_y_scaled = floor(size_y * scale);
-
- cv::Mat patch_gray = get_subwindow(input_gray, cx, cy, size_x_scaled, size_y_scaled);
- cv::Mat patch_rgb = get_subwindow(input_rgb, cx, cy, size_x_scaled, size_y_scaled);
-
- // resize to default size
- if (scale > 1.) {
- // if we downsample use INTER_AREA interpolation
- cv::resize(patch_gray, patch_gray, cv::Size(size_x, size_y), 0., 0., cv::INTER_AREA);
- } else {
- cv::resize(patch_gray, patch_gray, cv::Size(size_x, size_y), 0., 0., cv::INTER_LINEAR);
- }
-
// get hog(Histogram of Oriented Gradients) features
- vars.patch_feats = FHoG::extract(patch_gray, 2, p_cell_size, 9);
+ std::vector<cv::Mat> hog_feat = FHoG::extract(patch_gray, 2, p_cell_size, 9);
// get color rgb features (simple r,g,b channels)
std::vector<cv::Mat> color_feat;
- if ((m_use_color || m_use_cnfeat) && input_rgb.channels() == 3) {
- // resize to default size
- if (scale > 1.) {
- // if we downsample use INTER_AREA interpolation
- cv::resize(patch_rgb, patch_rgb, cv::Size(size_x / p_cell_size, size_y / p_cell_size), 0., 0., cv::INTER_AREA);
- } else {
- cv::resize(patch_rgb, patch_rgb, cv::Size(size_x / p_cell_size, size_y / p_cell_size), 0., 0., cv::INTER_LINEAR);
- }
- }
- if (m_use_color && input_rgb.channels() == 3) {
+ if (m_use_color && patch_rgb.channels() == 3) {
// use rgb color space
cv::Mat patch_rgb_norm;
patch_rgb.convertTo(patch_rgb_norm, CV_32F, 1. / 255., -0.5);
color_feat.insert(color_feat.end(), rgb.begin(), rgb.end());
}
- if (m_use_cnfeat && input_rgb.channels() == 3) {
+ if (m_use_cnfeat && patch_rgb.channels() == 3) {
std::vector<cv::Mat> cn_feat = CNFeat::extract(patch_rgb);
color_feat.insert(color_feat.end(), cn_feat.begin(), cn_feat.end());
}
- BIG_BATCH_OMP_ORDERED
- vars.patch_feats.insert(vars.patch_feats.end(), color_feat.begin(), color_feat.end());
- return;
+
+ hog_feat.insert(hog_feat.end(), color_feat.begin(), color_feat.end());
+ return hog_feat;
}
cv::Mat KCF_Tracker::gaussian_shaped_labels(double sigma, int dim1, int dim2)
float *row_ptr = labels.ptr<float>(j);
double y_s = y * y;
for (int x = range_x[0], i = 0; x < range_x[1]; ++x, ++i) {
- row_ptr[i] = float(std::exp(-0.5 * (y_s + x * x) / sigma_s)); //-1/2*e^((y^2+x^2)/sigma^2)
+ row_ptr[i] = std::exp(-0.5 * (y_s + x * x) / sigma_s); //-1/2*e^((y^2+x^2)/sigma^2)
}
}
if (x2 - x1 == 0 || y2 - y1 == 0)
patch = cv::Mat::zeros(height, width, CV_32FC1);
- else {
+ else
cv::copyMakeBorder(input(cv::Range(y1, y2), cv::Range(x1, x2)), patch, top, bottom, left, right,
cv::BORDER_REPLICATE);
- // imshow( "copyMakeBorder", patch);
- // cv::waitKey();
- }
// sanity check
assert(patch.cols == width && patch.rows == height);
return patch;
}
-void KCF_Tracker::gaussian_correlation(struct ThreadCtx &vars, const ComplexMat &xf, const ComplexMat &yf,
- double sigma, bool auto_correlation)
+void KCF_Tracker::geometric_transformations(cv::Mat &patch, int size_x, int size_y, int angle, bool allow_debug)
+{
+ if (m_use_angle) {
+ cv::Point2f center((patch.cols - 1) / 2., (patch.rows - 1) / 2.);
+ cv::Mat r = cv::getRotationMatrix2D(center, angle, 1.0);
+
+ cv::warpAffine(patch, patch, r, cv::Size(patch.cols, patch.rows), cv::INTER_LINEAR, cv::BORDER_REPLICATE);
+ }
+
+ // resize to default size
+ if (patch.channels() != 3) {
+ if (patch.cols / size_x > 1.) {
+ // if we downsample use INTER_AREA interpolation
+ cv::resize(patch, patch, cv::Size(size_x, size_y), 0., 0., cv::INTER_AREA);
+ } else {
+ cv::resize(patch, patch, cv::Size(size_x, size_y), 0., 0., cv::INTER_LINEAR);
+ }
+ } else {
+ if (patch.cols / size_x > 1.) {
+ // if we downsample use INTER_AREA interpolation
+ cv::resize(patch, patch, cv::Size(size_x / p_cell_size, size_y / p_cell_size), 0., 0., cv::INTER_AREA);
+ } else {
+ cv::resize(patch, patch, cv::Size(size_x / p_cell_size, size_y / p_cell_size), 0., 0., cv::INTER_LINEAR);
+ }
+ if (m_visual_debug && allow_debug) {
+ cv::Mat input_clone = patch.clone();
+ cv::resize(input_clone, input_clone, cv::Size(p_debug_image_size, p_debug_image_size), 0., 0.,
+ cv::INTER_LINEAR);
+
+ std::string angle_string = std::to_string(p_current_angle + angle);
+
+ cv::putText(input_clone, angle_string, cv::Point(1, input_clone.rows - 5), cv::FONT_HERSHEY_COMPLEX_SMALL,
+ 0.5, cv::Scalar(0, 255, 0), 1);
+
+ p_debug_subwindows.push_back(input_clone);
+ }
+ }
+}
+
+void KCF_Tracker::gaussian_correlation(struct ThreadCtx &vars, const ComplexMat &xf, const ComplexMat &yf, double sigma,
+ bool auto_correlation)
{
- #ifdef CUFFT
- xf.sqr_norm(vars.xf_sqr_norm.deviceMem());
- if (!auto_correlation) yf.sqr_norm(vars.yf_sqr_norm.deviceMem());
- #else
- xf.sqr_norm(vars.xf_sqr_norm.hostMem());
+ xf.sqr_norm(vars.xf_sqr_norm);
if (auto_correlation) {
vars.yf_sqr_norm.hostMem()[0] = vars.xf_sqr_norm.hostMem()[0];
} else {
- yf.sqr_norm(vars.yf_sqr_norm.hostMem());
+ yf.sqr_norm(vars.yf_sqr_norm);
}
- #endif
vars.xyf = auto_correlation ? xf.sqr_mag() : xf.mul2(yf.conj());
DEBUG_PRINTM(vars.xyf);
fft.inverse(vars.xyf, vars.ifft2_res, m_use_cuda ? vars.data_i_features.deviceMem() : nullptr, vars.stream);
#ifdef CUFFT
- if (auto_correlation)
- cuda_gaussian_correlation(vars.data_i_features.deviceMem(), vars.gauss_corr_res.deviceMem(), vars.xf_sqr_norm.deviceMem(), vars.xf_sqr_norm.deviceMem(),
- sigma, xf.n_channels, xf.n_scales, p_roi.height, p_roi.width, vars.stream);
- else
- cuda_gaussian_correlation(vars.data_i_features.deviceMem(), vars.gauss_corr_res.deviceMem(), vars.xf_sqr_norm.deviceMem(), vars.yf_sqr_norm.deviceMem(),
- sigma, xf.n_channels, xf.n_scales, p_roi.height, p_roi.width, vars.stream);
+ cuda_gaussian_correlation(vars.data_i_features.deviceMem(), vars.gauss_corr_res.deviceMem(),
+ vars.xf_sqr_norm.deviceMem(), vars.xf_sqr_norm.deviceMem(), sigma, xf.n_channels,
+ xf.n_scales, p_roi.height, p_roi.width, vars.stream);
#else
// ifft2 and sum over 3rd dimension, we dont care about individual channels
DEBUG_PRINTM(vars.ifft2_res);
A.at<float>(i, 0) = float(p_scales[i] * p_scales[i]);
A.at<float>(i, 1) = float(p_scales[i]);
A.at<float>(i, 2) = 1;
- fval.at<float>(i) = m_use_big_batch ? p_threadctxs.back().max_responses[i] : p_threadctxs[i].max_response;
+ #ifdef BIG_BATCH
+ fval.at<float>(i) = p_threadctxs.back().max_responses[i];
+ #else
+ fval.at<float>(i) = p_threadctxs[i].max_response;
+ #endif
}
} else {
// only from neighbours
p_scales[index - 1] * p_scales[index - 1], p_scales[index - 1], 1,
p_scales[index + 0] * p_scales[index + 0], p_scales[index + 0], 1,
p_scales[index + 1] * p_scales[index + 1], p_scales[index + 1], 1);
+ #ifdef BIG_BATCH
fval = (cv::Mat_<float>(3, 1) <<
- (m_use_big_batch ? p_threadctxs.back().max_responses[index - 1] : p_threadctxs[index - 1].max_response),
- (m_use_big_batch ? p_threadctxs.back().max_responses[index + 0] : p_threadctxs[index + 0].max_response),
- (m_use_big_batch ? p_threadctxs.back().max_responses[index + 1] : p_threadctxs[index + 1].max_response));
+ p_threadctxs.back().max_responses[index - 1],
+ p_threadctxs.back().max_responses[index + 0],
+ p_threadctxs.back().max_responses[index + 1]);
+ #else
+ fval = (cv::Mat_<float>(3, 1) <<
+ p_threadctxs[index - 1].max_response,
+ p_threadctxs[index + 0].max_response,
+ p_threadctxs[index + 1].max_response);
+ #endif
}
cv::Mat x;
projects / hercules2020 / kcf.git / commitdiff