Implement a specialized input visualisation.

src/InputLayerVisualisation.cpp

@@ -0,0 +1,103 @@
+#include <caffe/util/math_functions.hpp>
+#include <GL/glu.h>
+#include <opencv2/core/mat.hpp>
+#include <opencv2/core.hpp>
+#include "InputLayerVisualisation.hpp"
+#include "Range.hpp"
+
+using namespace fmri;
+using namespace std;
+
+/**
+ * Combine an arbitrary number of channels into an RGB image.
+ *
+ * If there are less than 3 channels, the first channel is repeated. This
+ * results in greyscale images for single-channel images. Any channels after
+ * the third are ignored.
+ *
+ * @param data Layer data to generate an image for.
+ * @return A normalized RGB image, stored in a vector.
+ */
+static vector<float> getRGBImage(const LayerData &data)
+{
+    vector<cv::Mat> channels;
+    const int numPixels = data.shape()[2] * data.shape()[3];
+    for (auto i : Range(3)) {
+        if (i >= data.shape()[1]) {
+            channels.push_back(channels[0]);
+        }
+
+        cv::Mat channel(data.shape()[3], data.shape()[2], CV_32FC1);
+        copy(data.data() + i * numPixels, data.data() + (i + 1) * numPixels, channel.begin<float>());
+        channels.push_back(channel);
+    }
+
+    swap(channels[0], channels[2]);
+
+    cv::Mat outImage;
+    cv::merge(channels, outImage);
+
+    outImage = outImage.reshape(1);
+
+    vector<float> final(outImage.begin<float>(), outImage.end<float>());
+    rescale(final.begin(), final.end(), 0.f, 1.f);
+
+    return final;
+}
+
+InputLayerVisualisation::InputLayerVisualisation(const LayerData &data)
+{
+    CHECK_EQ(data.shape().size(), 4) << "Should be image-like-layer." << endl;
+    auto imageData = getRGBImage(data);
+
+    const auto images = data.shape()[0], channels = data.shape()[1], width = data.shape()[2], height = data.shape()[3];
+    CHECK_EQ(images, 1) << "Should be single image" << endl;
+
+    targetWidth = width / 5.f;
+    targetHeight = width / 5.f;
+
+    nodePositions_ = {0, targetHeight / 2, targetWidth / -2};
+    for (auto i : Range(3, 3 * channels)) {
+        nodePositions_.push_back(nodePositions_[i % 3]);
+    }
+
+    texture.bind(GL_TEXTURE_2D);
+    // Set up (lack of) repetition
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_R, GL_CLAMP_TO_BORDER);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+
+    // Set up texture scaling
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_NEAREST); // Use mipmapping for scaling down
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); // Use nearest pixel when scaling up.
+    gluBuild2DMipmaps(GL_TEXTURE_2D, GL_RGB, width, height, GL_RGB, GL_FLOAT, imageData.data());
+}
+
+void InputLayerVisualisation::render()
+{
+    const float vertices[] = {
+            // Position, texture coordinates
+            0, 0, 0,
+            0, 0, -targetWidth,
+            0, targetHeight, -targetWidth,
+            0, targetHeight, 0,
+    };
+
+    const float texCoords[] = {
+            0, 1,
+            1, 1,
+            1, 0,
+            0, 0,
+    };
+
+    glEnableClientState(GL_VERTEX_ARRAY);
+    glEnableClientState(GL_TEXTURE_COORD_ARRAY);
+    glVertexPointer(3, GL_FLOAT, 0, vertices);
+    glTexCoordPointer(2, GL_FLOAT, 0, texCoords);
+    glEnable(GL_TEXTURE_2D);
+    glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE);
+    texture.bind(GL_TEXTURE_2D);
+    glDrawArrays(GL_QUADS, 0, 4);
+    glDisable(GL_TEXTURE_2D);
+    glDisableClientState(GL_TEXTURE_COORD_ARRAY);
+    glDisableClientState(GL_VERTEX_ARRAY);
+}

src/InputLayerVisualisation.hpp

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "LayerData.hpp"
+#include "LayerVisualisation.hpp"
+#include "Texture.hpp"
+
+namespace fmri
+{
+    class InputLayerVisualisation : public LayerVisualisation
+    {
+    public:
+        explicit InputLayerVisualisation(const LayerData &data);
+
+        void render() override;
+
+    private:
+        Texture texture;
+        float targetWidth;
+        float targetHeight;
+
+    };
+}

src/glutils.cpp

@@ -8,7 +8,6 @@
 #include <thread>
 #include "glutils.hpp"
 #include "Range.hpp"
-#include "Texture.hpp"
 
 using namespace fmri;
 using namespace std;

src/main.cpp

@@ -112,7 +112,7 @@ static void updateVisualisers()
     LayerVisualisation* prevVisualisation = nullptr;
 
     for (LayerData &layer : *rendererData.currentData) {
-        LayerVisualisation* visualisation = getVisualisationForLayer(layer);
+        LayerVisualisation* visualisation = getVisualisationForLayer(layer, rendererData.layerInfo.at(layer.name()));
         if (prevState && prevVisualisation && visualisation) {
             auto interaction = getActivityAnimation(*prevState, layer, rendererData.layerInfo.at(layer.name()), prevVisualisation->nodePositions(), visualisation->nodePositions());
             rendererData.animations.emplace_back(interaction);

src/visualisations.cpp

@@ -7,6 +7,7 @@
 #include "FlatLayerVisualisation.hpp"
 #include "Range.hpp"
 #include "ActivityAnimation.hpp"
+#include "InputLayerVisualisation.hpp"
 
 using namespace fmri;
 using namespace std;
@@ -65,17 +66,27 @@ static EntryList deduplicate(const EntryList& entries)
     return result;
 }
 
-fmri::LayerVisualisation *fmri::getVisualisationForLayer(const fmri::LayerData &layer)
+fmri::LayerVisualisation *fmri::getVisualisationForLayer(const fmri::LayerData &data, const fmri::LayerInfo &info)
 {
-    switch (layer.shape().size()) {
-        case 2:
-            return new FlatLayerVisualisation(layer, FlatLayerVisualisation::Ordering::SQUARE);
-
-        case 4:
-            return new MultiImageVisualisation(layer);
+    switch (info.type()) {
+        case LayerInfo::Type::Input:
+            if (data.shape().size() == 4) {
+                return new InputLayerVisualisation(data);
+            } else {
+                return new FlatLayerVisualisation(data, FlatLayerVisualisation::Ordering::SQUARE);
+            }
 
         default:
-            return new DummyLayerVisualisation();
+            switch (data.shape().size()) {
+                case 2:
+                    return new FlatLayerVisualisation(data, FlatLayerVisualisation::Ordering::SQUARE);
+
+                case 4:
+                    return new MultiImageVisualisation(data);
+
+                default:
+                    return new DummyLayerVisualisation();
+            }
     }
 }
 

src/visualisations.hpp

@@ -9,10 +9,10 @@ namespace fmri {
     /**
      * Generate a static visualisation of a layer state.
      *
-     * @param layer
+     * @param data
      * @return A (possibly empty) visualisation. The caller is responsible for deallocating.
      */
-    LayerVisualisation* getVisualisationForLayer(const LayerData& layer);
+    fmri::LayerVisualisation *getVisualisationForLayer(const fmri::LayerData &data, const fmri::LayerInfo &info);
 
     Animation * getActivityAnimation(const fmri::LayerData &prevState, const fmri::LayerData &curState,
                                      const fmri::LayerInfo &layer, const vector<float> &prevPositions,