From 265bc61b98418f5e0ff53fba3b5e01a0c8234405 Mon Sep 17 00:00:00 2001
From: Bert Peters <bert@bertptrs.nl>
Date: Thu, 12 Apr 2018 16:52:09 +0200
Subject: [PATCH] Large refactor of texture loading.

In this new implementation, everything is loaded on a separate thread,
but the textures are initialized on the main OpenGL thread.

This is all to work around the fact that you cannot create a separate
GL context in GLUT.
---
 src/fmri/Drawable.cpp                  |  5 ++
 src/fmri/Drawable.hpp                  |  1 +
 src/fmri/ImageInteractionAnimation.cpp |  9 ++-
 src/fmri/ImageInteractionAnimation.hpp |  3 +-
 src/fmri/InputLayerVisualisation.cpp   | 31 ++++++----
 src/fmri/InputLayerVisualisation.hpp   |  7 ++-
 src/fmri/MultiImageVisualisation.cpp   | 16 +++--
 src/fmri/MultiImageVisualisation.hpp   |  2 +
 src/fmri/PoolingLayerAnimation.cpp     | 10 ++-
 src/fmri/PoolingLayerAnimation.hpp     |  1 +
 src/fmri/RenderingState.cpp            | 15 +++--
 src/fmri/RenderingState.hpp            |  2 +
 src/fmri/Texture.cpp                   | 86 +++++++++++++++++++++-----
 src/fmri/Texture.hpp                   | 23 ++++---
 14 files changed, 158 insertions(+), 53 deletions(-)

diff --git a/src/fmri/Drawable.cpp b/src/fmri/Drawable.cpp
index a5d3997..41c8f32 100644
--- a/src/fmri/Drawable.cpp
+++ b/src/fmri/Drawable.cpp
@@ -14,3 +14,8 @@ void fmri::Drawable::patchTransparancy()
         (*it)[3] = alpha;
     }
 }
+
+void fmri::Drawable::glLoad()
+{
+    // Do nothing
+}
diff --git a/src/fmri/Drawable.hpp b/src/fmri/Drawable.hpp
index da37a95..60f2b31 100644
--- a/src/fmri/Drawable.hpp
+++ b/src/fmri/Drawable.hpp
@@ -12,6 +12,7 @@ namespace fmri
 
         virtual void draw(float time) = 0;
         virtual void patchTransparancy();
+        virtual void glLoad();
 
     protected:
         std::vector<Color> colorBuffer;
diff --git a/src/fmri/ImageInteractionAnimation.cpp b/src/fmri/ImageInteractionAnimation.cpp
index 535eebc..be22dbb 100644
--- a/src/fmri/ImageInteractionAnimation.cpp
+++ b/src/fmri/ImageInteractionAnimation.cpp
@@ -15,7 +15,7 @@ void ImageInteractionAnimation::draw(float step)
 
 ImageInteractionAnimation::ImageInteractionAnimation(const DType *data, const std::vector<int> &shape, const std::vector<float> &prevPositions,
                                                      const std::vector<float> &curPositions) :
-        texture(loadTexture(data, shape[2], shape[1] * shape[3], shape[1])),
+        texture(data, shape[2], shape[1] * shape[3], GL_LUMINANCE, shape[1]),
         startingPositions(MultiImageVisualisation::getVertices(prevPositions)),
         deltas(MultiImageVisualisation::getVertices(curPositions)),
         textureCoordinates(MultiImageVisualisation::getTexCoords(shape[1]))
@@ -26,3 +26,10 @@ ImageInteractionAnimation::ImageInteractionAnimation(const DType *data, const st
         deltas[i] = LAYER_X_OFFSET;
     }
 }
+
+void ImageInteractionAnimation::glLoad()
+{
+    Drawable::glLoad();
+
+    texture.configure(GL_TEXTURE_2D);
+}
diff --git a/src/fmri/ImageInteractionAnimation.hpp b/src/fmri/ImageInteractionAnimation.hpp
index a716e16..91c3f76 100644
--- a/src/fmri/ImageInteractionAnimation.hpp
+++ b/src/fmri/ImageInteractionAnimation.hpp
@@ -11,7 +11,8 @@ namespace fmri
     public:
         ImageInteractionAnimation(const DType *data, const std::vector<int> &shape, const std::vector<float> &prevPositions,
                                           const std::vector<float> &curPositions);
-        virtual void draw(float step);
+        void draw(float step) override;
+        void glLoad() override;
 
     private:
         Texture texture;
diff --git a/src/fmri/InputLayerVisualisation.cpp b/src/fmri/InputLayerVisualisation.cpp
index e083f17..96609ac 100644
--- a/src/fmri/InputLayerVisualisation.cpp
+++ b/src/fmri/InputLayerVisualisation.cpp
@@ -19,8 +19,10 @@ using namespace std;
  * @param data Layer data to generate an image for.
  * @return A normalized RGB image, stored in a vector.
  */
-static vector<float> getRGBImage(const LayerData &data)
+static unique_ptr<float[]> getRGBImage(const LayerData &data)
 {
+    CHECK_EQ(data.shape().size(), 4) << "Should be image-like-layer.";
+    CHECK_EQ(data.shape()[0], 1) << "Should be a single image";
     vector<cv::Mat> channels;
     const int numPixels = data.shape()[2] * data.shape()[3];
     for (auto i : Range(3)) {
@@ -29,7 +31,7 @@ static vector<float> getRGBImage(const LayerData &data)
         }
 
         cv::Mat channel(data.shape()[3], data.shape()[2], CV_32FC1);
-        copy(data.data() + i * numPixels, data.data() + (i + 1) * numPixels, channel.begin<float>());
+        copy_n(data.data() + i * numPixels, numPixels, channel.begin<float>());
         channels.push_back(channel);
     }
 
@@ -40,19 +42,18 @@ static vector<float> getRGBImage(const LayerData &data)
 
     outImage = outImage.reshape(1);
 
-    vector<float> final(outImage.begin<float>(), outImage.end<float>());
-    rescale(final.begin(), final.end(), 0.f, 1.f);
+    auto final = make_unique<float[]>(data.numEntries());
+    std::copy_n(outImage.begin<float>(), data.numEntries(), final.get());
 
     return final;
 }
 
-InputLayerVisualisation::InputLayerVisualisation(const LayerData &data)
+InputLayerVisualisation::InputLayerVisualisation(const LayerData &data) :
+        width(data.shape().at(2)),
+        height(data.shape().at(3)),
+        texture(getRGBImage(data), width, height, GL_RGB)
 {
-    CHECK_EQ(data.shape().size(), 4) << "Should be image-like-layer." << endl;
-    auto imageData = getRGBImage(data);
-
-    const auto images = data.shape()[0], channels = data.shape()[1], width = data.shape()[2], height = data.shape()[3];
-    CHECK_EQ(images, 1) << "Should be single image" << endl;
+    const auto channels = data.shape()[1];
 
     targetWidth = width / 5.f;
     targetHeight = width / 5.f;
@@ -61,9 +62,6 @@ InputLayerVisualisation::InputLayerVisualisation(const LayerData &data)
     for (auto i : Range(3, 3 * channels)) {
         nodePositions_.push_back(nodePositions_[i % 3]);
     }
-
-    texture.configure(GL_TEXTURE_2D);
-    gluBuild2DMipmaps(GL_TEXTURE_2D, GL_RGB, width, height, GL_RGB, GL_FLOAT, imageData.data());
 }
 
 void InputLayerVisualisation::draw(float)
@@ -86,3 +84,10 @@ void InputLayerVisualisation::draw(float)
 
     drawImageTiles(4, vertices, texCoords, texture, alpha);
 }
+
+void InputLayerVisualisation::glLoad()
+{
+    Drawable::glLoad();
+
+    texture.configure(GL_TEXTURE_2D);
+}
diff --git a/src/fmri/InputLayerVisualisation.hpp b/src/fmri/InputLayerVisualisation.hpp
index ee13570..befba04 100644
--- a/src/fmri/InputLayerVisualisation.hpp
+++ b/src/fmri/InputLayerVisualisation.hpp
@@ -13,10 +13,15 @@ namespace fmri
 
         void draw(float time) override;
 
+        void glLoad() override;
+
     private:
-        Texture texture;
         float targetWidth;
         float targetHeight;
+        int width;
+        int height;
+        Texture texture;
+        std::vector<float> textureBuffer;
 
     };
 }
diff --git a/src/fmri/MultiImageVisualisation.cpp b/src/fmri/MultiImageVisualisation.cpp
index 0154976..887de06 100644
--- a/src/fmri/MultiImageVisualisation.cpp
+++ b/src/fmri/MultiImageVisualisation.cpp
@@ -6,19 +6,16 @@
 using namespace fmri;
 using namespace std;
 
-MultiImageVisualisation::MultiImageVisualisation(const fmri::LayerData &layer)
+MultiImageVisualisation::MultiImageVisualisation(const fmri::LayerData &layer) :
+    texture(layer.data(), layer.shape().at(2), layer.shape().at(3) * layer.shape().at(1), GL_LUMINANCE, layer.shape().at(1))
 {
     auto dimensions = layer.shape();
-    CHECK_EQ(4, dimensions.size()) << "Should be image-like layer";
 
     const auto images = dimensions[0],
-            channels = dimensions[1],
-            width = dimensions[2],
-            height = dimensions[3];
+            channels = dimensions[1];
 
     CHECK_EQ(1, images) << "Only single input image is supported" << endl;
 
-    texture = loadTexture(layer.data(), width, channels * height, channels);
     initNodePositions<Ordering::SQUARE>(channels, 3);
     vertexBuffer = getVertices(nodePositions_);
     texCoordBuffer = getTexCoords(channels);
@@ -65,3 +62,10 @@ std::vector<float> MultiImageVisualisation::getTexCoords(int n)
 
     return coords;
 }
+
+void MultiImageVisualisation::glLoad()
+{
+    Drawable::glLoad();
+
+    texture.configure(GL_TEXTURE_2D);
+}
diff --git a/src/fmri/MultiImageVisualisation.hpp b/src/fmri/MultiImageVisualisation.hpp
index 2e861d0..f72fb92 100644
--- a/src/fmri/MultiImageVisualisation.hpp
+++ b/src/fmri/MultiImageVisualisation.hpp
@@ -22,6 +22,8 @@ namespace fmri
 
         void draw(float time) override;
 
+        void glLoad() override;
+
         static vector<float> getVertices(const std::vector<float> &nodePositions, float scaling = 1);
         static std::vector<float> getTexCoords(int n);
 
diff --git a/src/fmri/PoolingLayerAnimation.cpp b/src/fmri/PoolingLayerAnimation.cpp
index 60b3fb7..ba03577 100644
--- a/src/fmri/PoolingLayerAnimation.cpp
+++ b/src/fmri/PoolingLayerAnimation.cpp
@@ -41,5 +41,13 @@ Texture PoolingLayerAnimation::loadTextureForData(const LayerData &data)
     CHECK_EQ(data.shape()[0], 1) << "Only single images supported";
 
     auto channels = data.shape()[1], width = data.shape()[2], height = data.shape()[3];
-    return loadTexture(data.data(), width, height * channels, channels);
+    return Texture(data.data(), width, height * channels, GL_LUMINANCE, channels);
+}
+
+void PoolingLayerAnimation::glLoad()
+{
+    Drawable::glLoad();
+
+    original.configure(GL_TEXTURE_2D);
+    downSampled.configure(GL_TEXTURE_2D);
 }
diff --git a/src/fmri/PoolingLayerAnimation.hpp b/src/fmri/PoolingLayerAnimation.hpp
index 6f29712..8d6939e 100644
--- a/src/fmri/PoolingLayerAnimation.hpp
+++ b/src/fmri/PoolingLayerAnimation.hpp
@@ -14,6 +14,7 @@ namespace fmri
                                       const std::vector<float> &curPositions);
 
         void draw(float timeStep) override;
+        void glLoad() override;
 
     private:
         Texture original;
diff --git a/src/fmri/RenderingState.cpp b/src/fmri/RenderingState.cpp
index 5f50a53..9e5b511 100644
--- a/src/fmri/RenderingState.cpp
+++ b/src/fmri/RenderingState.cpp
@@ -223,9 +223,7 @@ void RenderingState::loadSimulationData(const std::map<string, LayerInfo> &info,
 void RenderingState::queueUpdate()
 {
     loadingFuture = std::async(std::launch::async, []() {
-        // Currently causes a segfault, due to threaded OpenGL.
-        // Possible solution: don't do GL things while loading.
-        //RenderingState::instance().updateVisualisers();
+        RenderingState::instance().updateVisualisers();
     });
     isLoading = true;
 }
@@ -456,7 +454,7 @@ float RenderingState::layerAlpha() const
 void RenderingState::idleFunc()
 {
     if (isLoading && loadingFuture.valid()) {
-        auto result = loadingFuture.wait_for(std::chrono::milliseconds(40));
+        auto result = loadingFuture.wait_for(std::chrono::milliseconds(16));
         switch (result) {
             case std::future_status::deferred:
                 LOG(ERROR) << "loading status was deferred, invalid state!";
@@ -468,7 +466,8 @@ void RenderingState::idleFunc()
 
             case std::future_status::ready:
                 loadingFuture.get();
-                //isLoading = false;
+                loadGLItems();
+                isLoading = false;
                 break;
         }
     } else {
@@ -484,3 +483,9 @@ void RenderingState::idleFunc()
     }
     glutPostRedisplay();
 }
+
+void RenderingState::loadGLItems()
+{
+    std::for_each(layerVisualisations.begin(), layerVisualisations.end(), [](auto& x) { x->glLoad(); });
+    std::for_each(interactionAnimations.begin(), interactionAnimations.end(), [](auto& x) { if (x) x->glLoad(); });
+}
diff --git a/src/fmri/RenderingState.hpp b/src/fmri/RenderingState.hpp
index 1de528c..eb7c433 100644
--- a/src/fmri/RenderingState.hpp
+++ b/src/fmri/RenderingState.hpp
@@ -102,5 +102,7 @@ namespace fmri
         void renderLoadingScreen() const;
 
         void renderVisualisation(float time) const;
+
+        void loadGLItems();
     };
 }
diff --git a/src/fmri/Texture.cpp b/src/fmri/Texture.cpp
index b34b49f..c189804 100644
--- a/src/fmri/Texture.cpp
+++ b/src/fmri/Texture.cpp
@@ -1,17 +1,11 @@
 #include <algorithm>
+#include <glog/logging.h>
+#include <GL/glu.h>
 #include "Texture.hpp"
+#include "utils.hpp"
 
 using namespace fmri;
 
-Texture::Texture() noexcept
-{
-    glGenTextures(1, &id);
-}
-
-Texture::Texture(GLuint id) noexcept : id(id)
-{
-}
-
 Texture::~Texture()
 {
     if (id != 0) {
@@ -22,21 +16,23 @@ Texture::~Texture()
 Texture &Texture::operator=(Texture && other) noexcept
 {
     std::swap(id, other.id);
+    std::swap(width, other.width);
+    std::swap(height, other.height);
+    std::swap(data, other.data);
+    std::swap(format, other.format);
     return *this;
 }
 
-Texture::Texture(Texture && other) noexcept
-{
-    std::swap(id, other.id);
-}
-
 void Texture::bind(GLenum target) const
 {
+    CHECK_NE(id, 0) << "Texture doesn't hold a reference!";
     glBindTexture(target, id);
 }
 
-void Texture::configure(GLenum target) const
+void Texture::configure(GLenum target)
 {
+    ensureReference();
+    CHECK(data) << "No valid data to configure with";
     bind(target);
     const float color[] = {1, 0, 1}; // Background color for textures.
 
@@ -48,4 +44,64 @@ void Texture::configure(GLenum target) const
     // Set up texture scaling
     glTexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_NEAREST); // Use mipmapping for scaling down
     glTexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_NEAREST); // Use nearest pixel when scaling up.
+    gluBuild2DMipmaps(target, format, width, height, format, GL_FLOAT, data.get());
+    data.reset(); // Release data buffer.
+}
+
+void Texture::ensureReference()
+{
+    if (id == 0) {
+        glGenTextures(1, &id);
+        CHECK_NE(id, 0) << "Failed to allocate a texture.";
+    }
+}
+
+Texture::Texture(const float *data, int width, int height, GLuint format, int subImages) :
+        Texture(std::make_unique<float[]>(width * height), width, height, format, subImages)
+{
+    std::copy_n(data, width * height, this->data.get());
+    preCalc(subImages);
+}
+
+Texture::Texture(std::unique_ptr<float[]> &&data, int width, int height, GLuint format, int subImages) :
+    id(0),
+    width(width),
+    height(height),
+    format(format),
+    data(std::move(data))
+{
+    preCalc(subImages);
+}
+
+void Texture::preCalc(int subImages)
+{
+    CHECK_EQ(height % subImages, 0) << "Image should be properly divisible!";
+
+    // Rescale images
+    const auto step = width * height * getStride() / subImages;
+    auto cur = data.get();
+    for (auto i = 0; i < subImages; ++i) {
+        rescale(cur, cur + step, 0, 1);
+        std::advance(cur, step);
+    }
+}
+
+int Texture::getStride()
+{
+    switch (format) {
+        case GL_RGB:
+            return 3;
+
+        case GL_LUMINANCE:
+            return 1;
+
+        default:
+            LOG(ERROR) << "Stride unknown for format: " << format;
+            exit(1);
+    }
+}
+
+Texture::Texture() noexcept :
+id(0)
+{
 }
diff --git a/src/fmri/Texture.hpp b/src/fmri/Texture.hpp
index 3e4eb63..2a10cc3 100644
--- a/src/fmri/Texture.hpp
+++ b/src/fmri/Texture.hpp
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <memory>
 #include <GL/gl.h>
 
 namespace fmri
@@ -13,19 +14,12 @@ namespace fmri
     class Texture
     {
     public:
-        /**
-         * Allocate a new texture
-         */
         Texture() noexcept;
+        Texture(const float* data, int width, int height, GLuint format, int subImages = 1);
+        Texture(std::unique_ptr<float[]> &&data, int width, int height, GLuint format, int subImages = 1);
         Texture(Texture &&) noexcept;
         Texture(const Texture &) = delete;
 
-        /**
-         * Own an existing texture
-         * @param id original texture ID.
-         */
-        explicit Texture(GLuint id) noexcept;
-
         ~Texture();
 
         Texture &operator=(Texture &&) noexcept;
@@ -36,10 +30,19 @@ namespace fmri
          * @param target valid target for glBindTexture.
          */
         void bind(GLenum target) const;
-        void configure(GLenum target) const;
+        void configure(GLenum target);
 
     private:
         GLuint id;
+        int width;
+        int height;
+        GLuint format;
+        std::unique_ptr<float[]> data;
 
+        void ensureReference();
+
+        void preCalc(int subImages);
+
+        int getStride();
     };
 }