(Mat)DynMem updates and related changes

DynMem has device pointer only when compiling for CUDA.

diff --git a/src/dynmem.hpp b/src/dynmem.hpp
index 09e16e336d498bdb4cecadbdbfbc11903bb2e2bc..4c5199e1273b88cc3740f9c7ba5275a0b9d74a75 100644 (file)
--- a/src/dynmem.hpp
+++ b/src/dynmem.hpp
@@ -13,14 +13,14 @@
 #endif
 
 template <typename T> class DynMem_ {
-    T *ptr = nullptr;
+  private:
+    T *ptr_h = nullptr;
+#ifdef CUFFT
     T *ptr_d = nullptr;
-
+#endif
   public:
     typedef T type;
-
-    DynMem_()
-    {}
+    DynMem_() {}
     DynMem_(size_t size)
     {
 #ifdef CUFFT
@@ -28,39 +28,44 @@ template <typename T> class DynMem_ {
         CudaSafeCall(
             cudaHostGetDevicePointer(reinterpret_cast<void **>(&this->ptr_d), reinterpret_cast<void *>(this->ptr), 0));
 #else
-        this->ptr = new float[size];
+        this->ptr_h = new float[size];
 #endif
     }
     DynMem_(DynMem_&& other) {
-        this->ptr = other.ptr;
+        this->ptr_h = other.ptr_h;
+        other.ptr_h = nullptr;
+#ifdef CUFFT
         this->ptr_d = other.ptr_d;
-
-        other.ptr = nullptr;
         other.ptr_d = nullptr;
+#endif
     }
     ~DynMem_()
     {
 #ifdef CUFFT
         CudaSafeCall(cudaFreeHost(this->ptr));
 #else
-        delete[] this->ptr;
+        delete[] this->ptr_h;
 #endif
     }
-    T *hostMem() { return ptr; }
+    T *hostMem() { return ptr_h; }
+#ifdef CUFFT
     T *deviceMem() { return ptr_d; }
-
+#endif
     void operator=(DynMem_ &&rhs)
     {
-        this->ptr = rhs.ptr;
+        this->ptr_h = rhs.ptr_h;
+        rhs.ptr_h = nullptr;
+#ifdef CUFFT
         this->ptr_d = rhs.ptr_d;
-
-        rhs.ptr = nullptr;
         rhs.ptr_d = nullptr;
+#endif
     }
 };
+
 typedef DynMem_<float> DynMem;
 
-class MatDynMem : public DynMem, public cv::Mat {
+
+class MatDynMem : protected DynMem, public cv::Mat {
   public:
     MatDynMem(cv::Size size, int type)
         : DynMem(size.area() * sizeof(DynMem::type) * CV_MAT_CN(type)), cv::Mat(size, type, hostMem())
@@ -68,6 +73,23 @@ class MatDynMem : public DynMem, public cv::Mat {
         assert((type & CV_MAT_DEPTH_MASK) == CV_32F);
     }
     MatDynMem(int height, int width, int type) { MatDynMem(cv::Size(width, height), type); }
+    MatDynMem(int ndims, const int *sizes, int type)
+        : DynMem(volume(ndims, sizes) * sizeof(DynMem::type) * CV_MAT_CN(type)), cv::Mat(ndims, sizes, type, hostMem())
+    {
+        assert((type & CV_MAT_DEPTH_MASK) == CV_32F);
+    }
+    void operator=(const cv::MatExpr &expr) {
+        static_cast<cv::Mat>(*this) = expr;
+    }
+
+  private:
+    static int volume(int ndims, const int *sizes)
+    {
+        int vol = 1;
+        for (int i = 0; i < ndims; i++)
+            vol *= sizes[i];
+        return vol;
+    }
 };
 
 #endif // DYNMEM_HPP

diff --git a/src/fft.h b/src/fft.h
index cc1393d45abcd3a96138ac90c52874e336befe5c..b165ef466fec20cbe1fabd55fbb60e8a73ab8d8e 100644 (file)
--- a/src/fft.h
+++ b/src/fft.h
@@ -4,6 +4,7 @@
 
 #include <opencv2/opencv.hpp>
 #include <vector>
+#include <cassert>
 
 #ifdef CUFFT
     #include "complexmat.cuh"
@@ -21,10 +22,10 @@ class Fft
 {
 public:
     virtual void init(unsigned width, unsigned height,unsigned num_of_feats, unsigned num_of_scales) = 0;
-    virtual void set_window(const cv::Mat & window) = 0;
-    virtual void forward(const cv::Mat & real_input, ComplexMat & complex_result, float *real_input_arr) = 0;
-    virtual void forward_window(std::vector<cv::Mat> patch_feats, ComplexMat & complex_result, cv::Mat & fw_all, float *real_input_arr) = 0;
-    virtual void inverse(ComplexMat &  complex_input, cv::Mat & real_result, float *real_result_arr) = 0;
+    virtual void set_window(const MatDynMem &window) = 0;
+    virtual void forward(const cv::Mat & real_input, ComplexMat & complex_result) = 0;
+    virtual void forward_window(MatDynMem &patch_feats_in, ComplexMat & complex_result, MatDynMem &tmp) = 0;
+    virtual void inverse(ComplexMat &  complex_input, MatDynMem & real_result) = 0;
     virtual ~Fft() = 0;
 
     static cv::Size freq_size(cv::Size space_size)
@@ -35,6 +36,14 @@ public:
 #endif
         return ret;
     }
+
+protected:
+    bool is_patch_feats_valid(const MatDynMem &patch_feats)
+    {
+        return patch_feats.dims == 3;
+               // && patch_feats.size[1] == width
+               // && patch_feats.size[2] == height
+    }
 };
 
 #endif // FFT_H

diff --git a/src/fft_cufft.cpp b/src/fft_cufft.cpp
index d3dadf1589e20350a359678cbef34744113d9cf9..cd53414bbf5921761bf725db9ae1b66a2e016224 100644 (file)
--- a/src/fft_cufft.cpp
+++ b/src/fft_cufft.cpp
@@ -114,7 +114,7 @@ void cuFFT::init(unsigned width, unsigned height, unsigned num_of_feats, unsigne
 #endif
 }
 
-void cuFFT::set_window(const cv::Mat &window)
+void cuFFT::set_window(const MatDynMem &window)
 {
     m_window = window;
 }
@@ -135,8 +135,7 @@ void cuFFT::forward(const cv::Mat &real_input, ComplexMat &complex_result, float
     return;
 }
 
-void cuFFT::forward_window(std::vector<cv::Mat> patch_feats, ComplexMat &complex_result, cv::Mat &fw_all,
-                           float *real_input_arr)
+void cuFFT::forward_window(MatDynMem &patch_feats_in, ComplexMat & complex_result, MatDynMem &tmp)
 {
     int n_channels = int(patch_feats.size());
 
@@ -162,7 +161,7 @@ void cuFFT::forward_window(std::vector<cv::Mat> patch_feats, ComplexMat &complex
     return;
 }
 
-void cuFFT::inverse(ComplexMat &complex_input, cv::Mat &real_result, float *real_result_arr)
+void cuFFT::inverse(ComplexMat &  complex_input, MatDynMem & real_result)
 {
     int n_channels = complex_input.n_channels;
     cufftComplex *in = reinterpret_cast<cufftComplex *>(complex_input.get_p_data());

diff --git a/src/fft_cufft.h b/src/fft_cufft.h
index 367b95a6e11876293dcbe574a2156d5cfe537000..9b2467f254c80f151572ba39af329c20368e697f 100644 (file)
--- a/src/fft_cufft.h
+++ b/src/fft_cufft.h
@@ -15,10 +15,10 @@ class cuFFT : public Fft
 {
 public:
     void init(unsigned width, unsigned height, unsigned num_of_feats, unsigned num_of_scales) override;
-    void set_window(const cv::Mat & window) override;
-    void forward(const cv::Mat & real_input, ComplexMat & complex_result, float *real_input_arr) override;
-    void forward_window(std::vector<cv::Mat> patch_feats, ComplexMat & complex_result, cv::Mat & fw_all, float *real_input_arr) override;
-    void inverse(ComplexMat &  complex_input, cv::Mat & real_result, float *real_result_arr) override;
+    void set_window(const MatDynMem &window) override;
+    void forward(const cv::Mat & real_input, ComplexMat & complex_result) override;
+    void forward_window(MatDynMem &patch_feats_in, ComplexMat & complex_result, MatDynMem &tmp) override;
+    void inverse(ComplexMat &  complex_input, MatDynMem & real_result) override;
     ~cuFFT() override;
 private:
     cv::Mat m_window;

diff --git a/src/fft_fftw.cpp b/src/fft_fftw.cpp
index 524480f48a750f3edc97da56121f0ec5832221c2..973c894b8ce66f4b34ac8fab4b3da09d67c267a5 100644 (file)
--- a/src/fft_fftw.cpp
+++ b/src/fft_fftw.cpp
@@ -167,15 +167,13 @@ void Fftw::init(unsigned width, unsigned height, unsigned num_of_feats, unsigned
 #endif
 }
 
-void Fftw::set_window(const cv::Mat &window)
+void Fftw::set_window(const MatDynMem &window)
 {
     m_window = window;
 }
 
-void Fftw::forward(const cv::Mat &real_input, ComplexMat &complex_result, float *real_input_arr)
+void Fftw::forward(const cv::Mat & real_input, ComplexMat & complex_result)
 {
-    (void)real_input_arr;
-
     if (BIG_BATCH_MODE && real_input.rows == int(m_height * m_num_of_scales)) {
         fftwf_execute_dft_r2c(plan_f_all_scales, reinterpret_cast<float *>(real_input.data),
                               reinterpret_cast<fftwf_complex *>(complex_result.get_p_data()));
@@ -186,18 +184,18 @@ void Fftw::forward(const cv::Mat &real_input, ComplexMat &complex_result, float
     return;
 }
 
-void Fftw::forward_window(std::vector<cv::Mat> patch_feats, ComplexMat &complex_result, cv::Mat &fw_all,
-                          float *real_input_arr)
+void Fftw::forward_window(MatDynMem &feat, ComplexMat & complex_result, MatDynMem &temp)
 {
-    (void)real_input_arr;
+    assert(is_patch_feats_valid(feat));
 
-    int n_channels = int(patch_feats.size());
+    int n_channels = feat.size[0];
     for (int i = 0; i < n_channels; ++i) {
-        cv::Mat in_roi(fw_all, cv::Rect(0, i * int(m_height), int(m_width), int(m_height)));
-        in_roi = patch_feats[uint(i)].mul(m_window);
+        cv::Mat feat_plane(feat.dims - 1, feat.size + 1, feat.cv::Mat::type(), feat.ptr<void>(i));
+        cv::Mat temp_plane(temp.dims - 1, temp.size + 1, temp.cv::Mat::type(), temp.ptr(i));
+        temp_plane = feat_plane.mul(m_window);
     }
 
-    float *in = reinterpret_cast<float *>(fw_all.data);
+    float *in = temp.ptr<float>();
     fftwf_complex *out = reinterpret_cast<fftwf_complex *>(complex_result.get_p_data());
 
     if (n_channels <= int(m_num_of_feats))
@@ -207,13 +205,11 @@ void Fftw::forward_window(std::vector<cv::Mat> patch_feats, ComplexMat &complex_
     return;
 }
 
-void Fftw::inverse(ComplexMat &complex_input, cv::Mat &real_result, float *real_result_arr)
+void Fftw::inverse(ComplexMat &  complex_input, MatDynMem & real_result)
 {
-    (void)real_result_arr;
-
     int n_channels = complex_input.n_channels;
     fftwf_complex *in = reinterpret_cast<fftwf_complex *>(complex_input.get_p_data());
-    float *out = reinterpret_cast<float *>(real_result.data);
+    float *out = real_result.ptr<float>();
 
     if (n_channels == 1)
         fftwf_execute_dft_c2r(plan_i_1ch, in, out);
@@ -225,7 +221,6 @@ void Fftw::inverse(ComplexMat &complex_input, cv::Mat &real_result, float *real_
         fftwf_execute_dft_c2r(plan_i_features, in, out);
 
     real_result = real_result / (m_width * m_height);
-    return;
 }
 
 Fftw::~Fftw()

diff --git a/src/fft_fftw.h b/src/fft_fftw.h
index 75d2f28e53bc0591bfc72a52141a0cb3a366a7a2..97641f883939876996bae0220175a9a9b89af396 100644 (file)
--- a/src/fft_fftw.h
+++ b/src/fft_fftw.h
@@ -19,10 +19,10 @@ class Fftw : public Fft
 public:
     Fftw();
     void init(unsigned width, unsigned height, unsigned num_of_feats, unsigned num_of_scales) override;
-    void set_window(const cv::Mat & window) override;
-    void forward(const cv::Mat & real_input, ComplexMat & complex_result, float *real_input_arr) override;
-    void forward_window(std::vector<cv::Mat> patch_feats, ComplexMat & complex_result, cv::Mat & fw_all, float *real_input_arr) override;
-    void inverse(ComplexMat &  complex_input, cv::Mat & real_result, float *real_result_arr) override;
+    void set_window(const MatDynMem &window) override;
+    void forward(const cv::Mat & real_input, ComplexMat & complex_result) override;
+    void forward_window(MatDynMem &patch_feats_in, ComplexMat & complex_result, MatDynMem &tmp) override;
+    void inverse(ComplexMat &  complex_input, MatDynMem & real_result) override;
     ~Fftw() override;
 private:
     unsigned m_width, m_height, m_num_of_feats, m_num_of_scales;

diff --git a/src/fft_opencv.cpp b/src/fft_opencv.cpp
index a909f844c21a3121be2b2f893474371ae3e1e981..bb7675734177ce109125b428253021ba92d83c05 100644 (file)
--- a/src/fft_opencv.cpp
+++ b/src/fft_opencv.cpp
@@ -9,7 +9,7 @@ void FftOpencv::init(unsigned width, unsigned height, unsigned num_of_feats, uns
     std::cout << "FFT: OpenCV" << std::endl;
 }
 
-void FftOpencv::set_window(const cv::Mat &window)
+void FftOpencv::set_window(const MatDynMem &window)
 {
     m_window = window;
 }
@@ -24,8 +24,7 @@ void FftOpencv::forward(const cv::Mat &real_input, ComplexMat &complex_result, f
     return;
 }
 
-void FftOpencv::forward_window(std::vector<cv::Mat> patch_feats, ComplexMat &complex_result, cv::Mat &fw_all,
-                               float *real_input_arr)
+void FftOpencv::forward_window(MatDynMem &patch_feats_in, ComplexMat & complex_result, MatDynMem &tmp)
 {
     (void)real_input_arr;
     (void)fw_all;
@@ -39,7 +38,7 @@ void FftOpencv::forward_window(std::vector<cv::Mat> patch_feats, ComplexMat &com
     return;
 }
 
-void FftOpencv::inverse(ComplexMat &complex_input, cv::Mat &real_result, float *real_result_arr)
+void FftOpencv::inverse(ComplexMat &  complex_input, MatDynMem & real_result)
 {
     (void)real_result_arr;
 

diff --git a/src/fft_opencv.h b/src/fft_opencv.h
index 9254652829d6d65210eefcf858d2ef28ee585460..1239acdd3e2c62e687d39d89fddd3683c11d2cf5 100644 (file)
--- a/src/fft_opencv.h
+++ b/src/fft_opencv.h
@@ -8,10 +8,10 @@ class FftOpencv : public Fft
 {
 public:
     void init(unsigned width, unsigned height, unsigned num_of_feats, unsigned num_of_scales) override;
-    void set_window(const cv::Mat & window) override;
-    void forward(const cv::Mat & real_input, ComplexMat & complex_result, float *real_input_arr) override;
-    void forward_window(std::vector<cv::Mat> patch_feats, ComplexMat & complex_result, cv::Mat & fw_all, float *real_input_arr) override;
-    void inverse(ComplexMat &  complex_input, cv::Mat & real_result, float *real_result_arr) override;
+    void set_window(const MatDynMem &window) override;
+    void forward(const cv::Mat & real_input, ComplexMat & complex_result) override;
+    void forward_window(MatDynMem &patch_feats_in, ComplexMat & complex_result, MatDynMem &tmp) override;
+    void inverse(ComplexMat &  complex_input, MatDynMem & real_result) override;
     ~FftOpencv() override;
 private:
     cv::Mat m_window;

diff --git a/src/kcf.cpp b/src/kcf.cpp
index 85c8e49d4d0b4df8c4b84ee8501ed4a4cd98051c..4585aa0c22fb6df5e2fd77393725c6045196dca7 100644 (file)
--- a/src/kcf.cpp
+++ b/src/kcf.cpp
@@ -141,10 +141,6 @@ void KCF_Tracker::init(cv::Mat &img, const cv::Rect &bbox, int fit_size_x, int f
     p_roi.width = p_windows_size.width / p_cell_size;
     p_roi.height = p_windows_size.height / p_cell_size;
 
-    p_num_of_feats = 31;
-    if (m_use_color) p_num_of_feats += 3;
-    if (m_use_cnfeat) p_num_of_feats += 10;
-
     p_scales.clear();
     if (m_use_scale)
         for (int i = -int(p_num_scales) / 2; i <= int(p_num_scales) / 2; ++i)
@@ -209,16 +205,13 @@ void KCF_Tracker::init(cv::Mat &img, const cv::Rect &bbox, int fit_size_x, int f
     fft.set_window(cosine_window_function(p_roi.width, p_roi.height));
 
     // window weights, i.e. labels
-    fft.forward(
-        gaussian_shaped_labels(p_output_sigma, p_roi.width, p_roi.height), p_yf,
-        m_use_cuda ? p_rot_labels_data.deviceMem() : nullptr);
+    fft.forward(gaussian_shaped_labels(p_output_sigma, p_roi.width, p_roi.height), p_yf);
     DEBUG_PRINTM(p_yf);
 
     // obtain a sub-window for training initial model
     std::vector<cv::Mat> patch_feats = get_features(input_rgb, input_gray, p_pose.cx, p_pose.cy,
                                                     p_windows_size.width, p_windows_size.height);
-    fft.forward_window(patch_feats, p_model_xf, d.threadctxs.front().fw_all,
-                       m_use_cuda ? d.threadctxs.front().data_features.deviceMem() : nullptr);
+    fft.forward_window(patch_feats, p_model_xf);
     DEBUG_PRINTM(p_model_xf);
 
     if (m_use_linearkernel) {
@@ -492,8 +485,12 @@ void KCF_Tracker::scale_track(ThreadCtx &vars, cv::Mat &input_rgb, cv::Mat &inpu
 
 // ****************************************************************************
 
-std::vector<cv::Mat> KCF_Tracker::get_features(cv::Mat & input_rgb, cv::Mat & input_gray, int cx, int cy, int size_x, int size_y, double scale)
+void KCF_Tracker::get_features(MatDynMem &result_3d, cv::Mat & input_rgb, cv::Mat & input_gray, int cx, int cy, int size_x, int size_y, double scale)
 {
+    assert(result_3d.size[0] == num_of_feats());
+    assert(result_3d.size[1] == size_x);
+    assert(result_3d.size[2] == size_y);
+
     int size_x_scaled = floor(size_x * scale);
     int size_y_scaled = floor(size_y * scale);
 

diff --git a/src/kcf.h b/src/kcf.h
index 258317d79d2bc4e431cb8d2229953f6e5e696bca..f0d79c181297504f86eb15d71faf68ce586bca81 100644 (file)
--- a/src/kcf.h
+++ b/src/kcf.h
@@ -125,8 +125,7 @@ private:
     double p_min_max_scale[2];
     std::vector<double> p_scales;
 
-    //for big batch
-    int p_num_of_feats;
+    const int p_num_of_feats = 31 + m_use_color ? 3 : 0 + m_use_cnfeat ? 10 : 0;
     cv::Size p_roi;
 
     Kcf_Tracker_Private &d;
@@ -164,7 +163,7 @@ private:
     std::unique_ptr<GaussianCorrelation> gaussian_correlation;
     cv::Mat circshift(const cv::Mat & patch, int x_rot, int y_rot);
     cv::Mat cosine_window_function(int dim1, int dim2);
-    std::vector<cv::Mat> get_features(cv::Mat & input_rgb, cv::Mat & input_gray, int cx, int cy, int size_x, int size_y, double scale = 1.);
+    void get_features(MatDynMem &feat_3d, cv::Mat & input_rgb, cv::Mat & input_gray, int cx, int cy, int size_x, int size_y, double scale = 1.);
     cv::Point2f sub_pixel_peak(cv::Point & max_loc, cv::Mat & response);
     double sub_grid_scale(uint index);
 

diff --git a/src/threadctx.hpp b/src/threadctx.hpp
index d3bccbacec7b55f62b9c470f82e38a615655e73d..b3bfe00f5d8e18a3dfd9c11129bac99abdf45fe9 100644 (file)
--- a/src/threadctx.hpp
+++ b/src/threadctx.hpp
@@ -61,9 +61,9 @@ public:
 
 #ifdef BIG_BATCH
     // Stores value of responses, location of maximal response and response maps for each scale
-    std::vector<double> max_responses{num_of_scales};
-    std::vector<cv::Point2i> max_locs{num_of_scales};
-    std::vector<cv::Mat> response_maps{num_of_scales};
+    std::vector<double> max_responses = std::vector<double>(num_of_scales);
+    std::vector<cv::Point2i> max_locs = std::vector<cv::Point2i>(num_of_scales);
+    std::vector<cv::Mat> response_maps = std::vector<cv::Mat>(num_of_scales);
 #else
     const double scale;
 #endif