diff --git a/src/InputLayerVisualisation.cpp b/src/InputLayerVisualisation.cpp index c2bfe40..ba576bc 100644 --- a/src/InputLayerVisualisation.cpp +++ b/src/InputLayerVisualisation.cpp @@ -68,7 +68,6 @@ InputLayerVisualisation::InputLayerVisualisation(const LayerData &data) void InputLayerVisualisation::render() { const float vertices[] = { - // Position, texture coordinates 0, 0, 0, 0, 0, -targetWidth, 0, targetHeight, -targetWidth, diff --git a/src/MultiImageVisualisation.hpp b/src/MultiImageVisualisation.hpp index c19aec7..e38348c 100644 --- a/src/MultiImageVisualisation.hpp +++ b/src/MultiImageVisualisation.hpp @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include "LayerVisualisation.hpp" @@ -12,17 +11,18 @@ namespace fmri class MultiImageVisualisation : public LayerVisualisation { public: - explicit MultiImageVisualisation(const LayerData&); - - void render() override; - - private: constexpr const static std::array BASE_VERTICES = { 0, -1, -1, 0, 1, -1, 0, 1, 1, 0, -1, 1, }; + + explicit MultiImageVisualisation(const LayerData&); + + void render() override; + + private: Texture texture; std::unique_ptr vertexBuffer; std::unique_ptr texCoordBuffer; diff --git a/src/PoolingLayerAnimation.cpp b/src/PoolingLayerAnimation.cpp new file mode 100644 index 0000000..a33f663 --- /dev/null +++ b/src/PoolingLayerAnimation.cpp @@ -0,0 +1,90 @@ +#include +#include +#include +#include "PoolingLayerAnimation.hpp" +#include "glutils.hpp" +#include "Range.hpp" +#include "MultiImageVisualisation.hpp" + +using namespace std; +using namespace fmri; + +PoolingLayerAnimation::PoolingLayerAnimation(const LayerData &prevData, const LayerData &curData, + const std::vector &prevPositions, + const std::vector &curPositions, float xDist) : + original(loadTextureForData(prevData)), + downSampled(loadTextureForData(curData)), + startingPositions(computePositions(prevPositions)), + deltas(startingPositions.size()) +{ + CHECK_EQ(prevPositions.size(), curPositions.size()) << "Layers should be same size. Caffe error?"; + + const float channels = curData.shape()[1]; + textureCoordinates.reserve(curPositions.size() / 3 * 4); + + const auto downScaling = sqrt( + static_cast(curData.shape()[2] * curData.shape()[3]) / (prevData.shape()[2] * prevData.shape()[3])); + + for (auto i : Range(prevPositions.size() / 3)) { + const array nodeTexCoords = { + 1, (i + 1) / channels, + 1, i / channels, + 0, i / channels, + 0, (i + 1) / channels, + }; + + for (auto coord : nodeTexCoords) { + textureCoordinates.push_back(coord); + } + } + + const auto targetPositions = computePositions(curPositions, downScaling); + caffe::caffe_sub(targetPositions.size(), targetPositions.data(), startingPositions.data(), deltas.data()); + + for (auto i = 0u; i < deltas.size(); i+=3) { + deltas[i] = xDist; + } +} + +void PoolingLayerAnimation::draw(float timeStep) +{ + vector vertexBuffer(deltas); + caffe::caffe_scal(vertexBuffer.size(), timeStep, vertexBuffer.data()); + caffe::caffe_add(startingPositions.size(), startingPositions.data(), vertexBuffer.data(), vertexBuffer.data()); + + glEnableClientState(GL_VERTEX_ARRAY); + glEnableClientState(GL_TEXTURE_COORD_ARRAY); + glEnable(GL_TEXTURE_2D); + original.bind(GL_TEXTURE_2D); + glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE); + glVertexPointer(3, GL_FLOAT, 0, vertexBuffer.data()); + glTexCoordPointer(2, GL_FLOAT, 0, textureCoordinates.data()); + glDrawArrays(GL_QUADS, 0, vertexBuffer.size() / 3); + glDisable(GL_TEXTURE_2D); + glDisableClientState(GL_VERTEX_ARRAY); + glDisableClientState(GL_TEXTURE_COORD_ARRAY); +} + +Texture PoolingLayerAnimation::loadTextureForData(const LayerData &data) +{ + CHECK_EQ(data.shape().size(), 4) << "Layer should be image-like"; + CHECK_EQ(data.shape()[0], 1) << "Only single images supported"; + + auto channels = data.shape()[1], width = data.shape()[2], height = data.shape()[3]; + return loadTexture(data.data(), width, height * channels, channels); +} + +vector PoolingLayerAnimation::computePositions(const vector &nodePositions, float scaling) +{ + vector positions; + positions.reserve(4 * nodePositions.size()); + + for (auto i : Range(nodePositions.size() / 3)) { + const float *pos = &nodePositions[3 * i]; + for (auto j : Range(MultiImageVisualisation::BASE_VERTICES.size())) { + positions.push_back(pos[j % 3] + MultiImageVisualisation::BASE_VERTICES[j] * scaling); + } + } + + return positions; +} diff --git a/src/PoolingLayerAnimation.hpp b/src/PoolingLayerAnimation.hpp new file mode 100644 index 0000000..57add47 --- /dev/null +++ b/src/PoolingLayerAnimation.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include "Animation.hpp" +#include "LayerData.hpp" +#include "Texture.hpp" + +namespace fmri +{ + class PoolingLayerAnimation : public Animation + { + public: + PoolingLayerAnimation(const LayerData& prevData, const LayerData& curData, const std::vector& prevPositions, const std::vector& curPositions, float xDist); + + void draw(float timeStep) override; + + private: + Texture original; + Texture downSampled; + std::vector startingPositions; + std::vector deltas; + std::vector textureCoordinates; + + static Texture loadTextureForData(const LayerData& data); + static std::vector computePositions(const std::vector &nodePositions, float scaling = 1); + }; +} diff --git a/src/visualisations.cpp b/src/visualisations.cpp index 719ea10..d5c094b 100644 --- a/src/visualisations.cpp +++ b/src/visualisations.cpp @@ -8,6 +8,7 @@ #include "Range.hpp" #include "ActivityAnimation.hpp" #include "InputLayerVisualisation.hpp" +#include "PoolingLayerAnimation.hpp" using namespace fmri; using namespace std; @@ -188,6 +189,9 @@ Animation * fmri::getActivityAnimation(const fmri::LayerData &prevState, const f case LayerInfo::Type::ReLU: return getReLUAnimation(prevState, curState, prevPositions, curPositions); + case LayerInfo::Type::Pooling: + return new PoolingLayerAnimation(prevState, curState, prevPositions, curPositions, -10); + default: return nullptr; }