Add Vulkan-OpenGL texture sharing, shader fixes

libraries/display-plugins/src/display-plugins/DrawTextureWithVisionSqueeze.slf

@@ -30,10 +30,10 @@ struct DrawTextureWithVisionSqueezeParams {
 };
 
 
-LAYOUT(binding=0) uniform sampler2D colorMap;
+TEXTURE(0, sampler2D, colorMap);
 
 // binding=1 must match drawTextureWithSqueezeParamsSlot in OpenGLDisplayPlugin.h
-LAYOUT(binding=1) uniform drawTextureWithSqueezeMappingParamsBuffer {
+UNIFORM_BUFFER(1, drawTextureWithSqueezeMappingParamsBuffer) {
     DrawTextureWithVisionSqueezeParams params;
 };
 

libraries/entities-renderer/src/paintStroke.slh

@@ -19,7 +19,7 @@
 // Hack comment to absorb the extra '//' scribe prepends
 
 #if !defined(GPU_SSBO_TRANSFORM_OBJECT)
-LAYOUT(binding=GPU_RESOURCE_BUFFER_SLOT0_TEXTURE) uniform samplerBuffer polylineVerticesBuffer;
+RESOURCE_BUFFER(GPU_RESOURCE_BUFFER_SLOT0_TEXTURE, ssbo0Buffer);
 PolylineVertex getPolylineVertex(int i) {
     int offset = 4 * i;
     PolylineVertex vertex;
@@ -30,7 +30,8 @@ PolylineVertex getPolylineVertex(int i) {
     return vertex;
 }
 #else
-LAYOUT_STD140(binding=GPU_RESOURCE_BUFFER_SLOT0_STORAGE) buffer polylineVerticesBuffer {
+// VKTODO: what is LAYOUT_STD140?
+RESOURCE_BUFFER(GPU_RESOURCE_BUFFER_SLOT0_STORAGE, polylineVerticesBuffer) {
     PolylineVertex _vertices[];
 };
 PolylineVertex getPolylineVertex(int i) {
@@ -39,7 +40,8 @@ PolylineVertex getPolylineVertex(int i) {
 }
 #endif
 
-LAYOUT_STD140(binding=0) uniform polylineDataBuffer {
+// VKTODO: what is LAYOUT_STD140?
+UNIFORM_BUFFER(0, polylineDataBuffer) {
     PolylineData _polylineData;
 };
 

libraries/gpu-vk/CMakeLists.txt

@@ -1,6 +1,6 @@
 set(TARGET_NAME gpu-vk)
 setup_hifi_library()
-link_hifi_libraries(shared shaders vk gpu)
+link_hifi_libraries(shared shaders vk gl gpu)
 GroupSources("src")
 
 target_vulkan()

libraries/gpu-vk/src/gpu/vk/VKBackend.cpp

@@ -1223,6 +1223,11 @@ void VKBackend::renderPassDraw(const Batch& batch) {
         case Batch::COMMAND_multiDrawIndexedIndirect: {
             // updates for draw calls
             ++_currentDraw;
+
+            /*if (_cache.pipelineState.pipeline->getProgram()->getShaders()[0]->getSource().name == "simple_procedural.vert") {
+                printf("simple_procedural.vert");
+                break;
+            }*/ // VKTODO: currently crashes on procedural shaders. I tried this as a workaround,but it didn't work.
             _cache.pipelineState.primitiveTopology = getPrimitiveTopologyFromCommand(*command, batch, *offset);
             updateInput();
             updateTransform(batch);
@@ -1413,7 +1418,7 @@ VKTexture* VKBackend::syncGPUObject(const Texture& texture) {
         // Return the texture object (if any) associated with the texture, without extensive logic
         // (external textures are
         return Backend::getGPUObject<GLTexture>(texture);*/
-        return nullptr;
+
         //return Parent::syncGPUObject(texturePointer);
     }
 
@@ -1430,6 +1435,10 @@ VKTexture* VKBackend::syncGPUObject(const Texture& texture) {
                 object = new VKAttachmentTexture(shared_from_this(), texture);
                 break;
 
+            case TextureUsageType::EXTERNAL:
+                object = new VKExternalTexture(shared_from_this(), texture);
+                break;
+
 #if FORCE_STRICT_TEXTURE
             case TextureUsageType::RESOURCE:
 #endif
@@ -1532,7 +1541,7 @@ VKTexture* VKBackend::syncGPUObject(const Texture& texture) {
         }
     }
 
-    _textures.insert(object);
+    _textures.insert(object); // VKTODO: shouldn't that only be inserted when new object is created?
     return object;
 }
 
@@ -1921,6 +1930,13 @@ void VKBackend::perFrameCleanup() {
     recycler.vkSurfacesKHR.clear();
     recycler.vkSurfacesKHR.reserve(capacityBeforeClear);
 
+    for (auto memory: recycler.vkDeviceMemories) {
+        vkFreeMemory(device, memory, nullptr);
+    }
+    capacityBeforeClear = recycler.vkDeviceMemories.capacity();
+    recycler.vkDeviceMemories.clear();
+    recycler.vkDeviceMemories.reserve(capacityBeforeClear);
+
     for (auto allocation : recycler.vmaAllocations) {
         vmaFreeMemory(vks::Allocation::getAllocator(), allocation);
     }

libraries/gpu-vk/src/gpu/vk/VKTexture.cpp

@@ -599,6 +599,206 @@ VKStrictResourceTexture::~VKStrictResourceTexture() {
     recycler.trashVmaAllocation(_vmaAllocation);
 }
 
+void VKExternalTexture::createTexture(VKBackend &backend) {
+    auto device = backend.getContext().device;
+
+    VkImageCreateInfo imageCI = vks::initializers::imageCreateInfo();
+    imageCI.imageType = VK_IMAGE_TYPE_2D;
+    imageCI.format = evalTexelFormatInternal(_gpuObject.getTexelFormat());
+    imageCI.extent.width = _gpuObject.getWidth();
+    imageCI.extent.height = _gpuObject.getHeight();
+    imageCI.extent.depth = 1;
+    imageCI.arrayLayers = _gpuObject.isArray() ? _gpuObject.getNumSlices() : 1;
+    imageCI.samples = VK_SAMPLE_COUNT_1_BIT;
+    imageCI.tiling = VK_IMAGE_TILING_OPTIMAL;
+    imageCI.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
+    if (_gpuObject.getType() == Texture::TEX_CUBE) {
+        imageCI.flags |= VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT;
+        imageCI.arrayLayers = 6;
+    }
+
+    _transferData.mipLevels = _gpuObject.getNumMips();
+    _transferData.width = _gpuObject.getWidth();
+    _transferData.height = _gpuObject.getHeight();
+
+    _transferData.buffer_size = 0;
+
+    /*for (uint16_t sourceMip = 0; sourceMip < _transferData.mipLevels; ++sourceMip) {
+        if (!_gpuObject.isStoredMipFaceAvailable(sourceMip)) {
+            continue;
+        }
+        _transferData.mips.emplace_back();
+        //VKTODO: error out if needed
+        size_t face_count = 1;
+        if (_gpuObject.getType() == Texture::TEX_CUBE) {
+            Q_ASSERT(_gpuObject.getNumFaces() == 6);
+            face_count = 6;
+        }else{
+            Q_ASSERT(_gpuObject.getNumFaces() == 1);
+        }
+
+        // Is conversion from RGB to RGBA needed?
+        bool needsAddingAlpha = false;
+        bool needsBGRToRGB = false;
+        auto storedFormat = _gpuObject.getStoredMipFormat();
+        auto texelFormat = _gpuObject.getTexelFormat();
+        if ((storedFormat.getSemantic() == gpu::BGRA || storedFormat.getSemantic() == gpu::SBGRA)
+            && !(texelFormat.getSemantic() == gpu::BGRA || texelFormat.getSemantic() == gpu::SBGRA)) {
+            needsBGRToRGB = true;
+        }
+        auto storedVkFormat = evalTexelFormatInternal(_gpuObject.getStoredMipFormat());
+        auto texelVkFormat = evalTexelFormatInternal(_gpuObject.getTexelFormat());
+        if (storedFormat.getDimension() != texelFormat.getDimension()) {
+            if (storedFormat.getDimension() == gpu::VEC3 && texelFormat.getDimension() == gpu::VEC4) {
+                // It's best to make sure that this is not happening in unexpected cases and causing bugs
+                Q_ASSERT((storedVkFormat == VK_FORMAT_R8G8B8_UNORM && texelVkFormat == VK_FORMAT_R8G8B8A8_UNORM)
+                         || (storedVkFormat == VK_FORMAT_R8G8B8_UNORM && texelVkFormat == VK_FORMAT_R8G8B8A8_SRGB));
+                needsAddingAlpha = true;
+            } else {
+                qDebug() << "Format mismatch, stored: " << storedVkFormat << " texel: " << texelVkFormat;
+                Q_ASSERT(false);
+            }
+        }
+
+        for (size_t face = 0; face < face_count; face++) {
+            auto dim = _gpuObject.evalMipDimensions(sourceMip);
+            auto mipData = _gpuObject.accessStoredMipFace(sourceMip, face);  // VKTODO: only one face for now
+            auto mipSize = _gpuObject.getStoredMipFaceSize(sourceMip, face);
+            if (mipData) {
+                TransferData::Mip mip{};
+                mip.offset = _transferData.buffer_size;
+                mip.size = mipSize;
+                mip.data = mipData;
+                mip.width = dim.x;
+                mip.height = dim.y;
+                mip.needsAddingAlpha = needsAddingAlpha;
+                mip.needsBGRToRGB = needsBGRToRGB;
+                if (needsAddingAlpha) {
+                    Q_ASSERT(mipSize % 3 == 0);
+                    _transferData.buffer_size += mipSize / 3 * 4;
+                } else {
+                    _transferData.buffer_size += mipSize;
+                }
+                _transferData.mips.back().push_back(mip);
+                // VKTODO auto texelFormat = evalTexelFormatInternal(_gpuObject.getStoredMipFormat());
+                //return copyMipFaceLinesFromTexture(targetMip, face, dim, 0, texelFormat.internalFormat, texelFormat.format, texelFormat.type, mipSize, mipData->readData());
+            } else {
+                qCDebug(gpu_vk_logging) << "Missing mipData level=" << sourceMip
+                                        << " face=" << 0 << " for texture " << _gpuObject.source().c_str();
+            }
+        }
+    }*/
+
+    imageCI.mipLevels = _transferData.mips.size();
+    VK_CHECK_RESULT(vkCreateImage(device->logicalDevice, &imageCI, nullptr, &_vkImage));
+
+    VkMemoryRequirements memoryRequirements;
+    vkGetImageMemoryRequirements(device->logicalDevice, _vkImage, &memoryRequirements);
+
+    _sharedMemorySize = memoryRequirements.size;
+
+    //VmaAllocationCreateInfo allocationCI = {};
+    //allocationCI.usage = VMA_MEMORY_USAGE_GPU_ONLY;
+    //qDebug() << "storedSize: " << _gpuObject.getStoredSize();
+    VkExportMemoryAllocateInfo exportMemoryAllocateInfo {};
+    exportMemoryAllocateInfo.sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO;
+#ifdef WIN32
+    exportMemoryAllocateInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_OPAQUE_WIN32_BIT;
+#else
+    exportMemoryAllocateInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
+#endif
+    VkMemoryAllocateInfo memoryAllocateInfo {};
+    memoryAllocateInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
+    memoryAllocateInfo.allocationSize = memoryRequirements.size;
+    memoryAllocateInfo.memoryTypeIndex = device->getMemoryType(memoryRequirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
+    memoryAllocateInfo.pNext = &exportMemoryAllocateInfo;
+
+    VK_CHECK_RESULT(vkAllocateMemory(device->logicalDevice, &memoryAllocateInfo, nullptr, &_sharedMemory));
+    VK_CHECK_RESULT(vkBindImageMemory(device->logicalDevice, _vkImage, _sharedMemory, 0));
+#ifdef WIN32
+    VkMemoryGetWin32HandleInfoKHR memoryGetWin32HandleInfoKHR {};
+    memoryGetWin32HandleInfoKHR.sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR;
+    memoryGetWin32HandleInfoKHR.memory = _sharedMemory,
+    memoryGetWin32HandleInfoKHR.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT_KHR;
+    VK_CHECK_RESULT(vkGetMemoryWin32HandleKHR(device->logicalDevice, &memoryGetWin32HandleInfoKHR, &_sharedHandle));
+#else
+    VkMemoryGetFdInfoKHR memoryGetFdInfoKHR {};
+    memoryGetFdInfoKHR.sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR;
+    memoryGetFdInfoKHR.memory = _sharedMemory;
+    memoryGetFdInfoKHR.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR;
+    VK_CHECK_RESULT(vkGetMemoryFdKHR(device->logicalDevice, &memoryGetFdInfoKHR, &_sharedFd));
+#endif
+}
+
+void VKExternalTexture::initGL(gpu::vk::VKBackend& backend) {
+    glCreateMemoryObjectsEXT(1, &_openGLMemoryObject);
+#ifdef WIN32
+    glImportMemoryWin32HandleEXT(_openGLMemoryObject, _sharedMemorySize, GL_HANDLE_TYPE_OPAQUE_WIN32_EXT, _sharedHandle);
+#else
+    glImportMemoryFdEXT(_openGLMemoryObject, _sharedMemorySize, GL_HANDLE_TYPE_OPAQUE_FD_EXT, _sharedFd);
+    _sharedFd = -1; // File descriptor can be used only once?
+#endif
+}
+
+void VKExternalTexture::transferGL(VKBackend &backend) {
+
+}
+
+void VKExternalTexture::postTransfer(VKBackend &backend) {
+    auto device = backend.getContext().device;
+    // Create sampler
+    VkSamplerCreateInfo samplerCreateInfo = {};
+    samplerCreateInfo.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
+    samplerCreateInfo.magFilter = VK_FILTER_LINEAR; // VKTODO
+    samplerCreateInfo.minFilter = VK_FILTER_LINEAR; // VKTODO
+    samplerCreateInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR;
+    samplerCreateInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_REPEAT;
+    samplerCreateInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_REPEAT;
+    samplerCreateInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_REPEAT;
+    samplerCreateInfo.mipLodBias = 0.0f;
+    samplerCreateInfo.compareOp = VK_COMPARE_OP_NEVER;
+    samplerCreateInfo.minLod = 0.0f;
+    samplerCreateInfo.maxLod = 0.0f;
+    samplerCreateInfo.maxAnisotropy = 1.0f;
+    VK_CHECK_RESULT(vkCreateSampler(device->logicalDevice, &samplerCreateInfo, nullptr, &_vkSampler));
+
+    // Create image view
+    VkImageViewCreateInfo viewCreateInfo = {};
+    viewCreateInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
+    viewCreateInfo.pNext = nullptr;
+    viewCreateInfo.viewType = getVKTextureType(_gpuObject);
+    viewCreateInfo.format = evalTexelFormatInternal(_gpuObject.getTexelFormat());
+    viewCreateInfo.subresourceRange = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 };
+    viewCreateInfo.subresourceRange.levelCount = 1;
+    if (_gpuObject.getType() == Texture::TEX_CUBE) {
+        viewCreateInfo.subresourceRange.layerCount = 6;
+    } else {
+        viewCreateInfo.subresourceRange.layerCount = 1;
+    }
+    viewCreateInfo.image = _vkImage;
+    VK_CHECK_RESULT(vkCreateImageView(device->logicalDevice, &viewCreateInfo, nullptr, &_vkImageView));
+};
+
+VKExternalTexture::~VKExternalTexture() {
+    auto backend = _backend.lock();
+    auto device = backend->getContext().device->logicalDevice;
+    auto &recycler = backend->getContext().recycler;
+    recycler.trashVkImageView(_vkImageView);
+    if (_vkSampler) {
+        recycler.trashVkSampler(_vkSampler);
+    }
+    recycler.trashVkImage(_vkImage);
+    recycler.trashVkDeviceMemory(_sharedMemory);
+}
+
+VkDescriptorImageInfo VKExternalTexture::getDescriptorImageInfo() {
+    VkDescriptorImageInfo result {};
+    result.sampler = _vkSampler;
+    result.imageLayout = _vkImageLayout;
+    result.imageView = _vkImageView;
+    return result;
+}
+
 /*Size VKTexture::copyMipFaceFromTexture(uint16_t sourceMip, uint16_t targetMip, uint8_t face) const {
     if (!_gpuObject.isStoredMipFaceAvailable(sourceMip)) {
         return 0;

libraries/gpu-vk/src/gpu/vk/VKTexture.h

@@ -9,9 +9,13 @@
 #define hifi_gpu_vk_VKTexture_h
 
 #include "VKShared.h"
-//#include "VKTextureTransfer.h"
+
+#include <gl/Config.h>
+#include <gl/GLHelpers.h>
+
 #include "VKBackend.h"
 
+
 namespace gpu { namespace vk {
 
 struct VKFilterMode {
@@ -308,6 +312,49 @@ class VKStrictResourceTexture: public VKFixedAllocationTexture {
     //VkDeviceMemory _vkDeviceMemory{ VK_NULL_HANDLE };
 };
 
+class VKExternalTexture: public VKTexture {
+    friend class VKBackend;
+
+public:
+    VKExternalTexture(const std::weak_ptr<VKBackend>& backend, const Texture& texture) :
+            VKTexture(backend, texture, false) {
+            VKBackend& vkBackend = *backend.lock();
+            VKExternalTexture::createTexture(vkBackend);
+            // VKTODO: maybe this needs to be done on OpenGL thread?
+            VKExternalTexture::initGL(vkBackend);
+            VKExternalTexture::transferGL(vkBackend);
+            VKExternalTexture::postTransfer(vkBackend);
+        }; // VKTODO
+    ~VKExternalTexture() override;
+
+protected:
+    Size size() const override { return _size; }
+
+    //VmaAllocation _vmaAllocation;
+    VkDeviceMemory _sharedMemory;
+    size_t _sharedMemorySize;
+    const Size _size{ 0 }; // VKTODO: how is this used?
+
+    void createTexture(VKBackend &backend) override;
+    void initGL(VKBackend &backend);
+    void transferGL(VKBackend &backend);
+    void transfer(VKBackend &backend) override {};
+    void postTransfer(VKBackend &backend) override;
+    VkDescriptorImageInfo getDescriptorImageInfo() override;
+    VkImageView _vkImageView { VK_NULL_HANDLE };
+    VkImageLayout _vkImageLayout {}; // VKTODO
+    VkSampler _vkSampler { VK_NULL_HANDLE };
+
+    // Shared texture properties
+#ifdef WIN32
+    HANDLE _sharedHandle { nullptr };  // Windows handle for shared texture memory.
+#else
+    int _sharedFd { -1 }; // File descriptor is used for sharing memory between Vulkan and OpenGL on Linux.
+#endif
+    GLuint _openGLMemoryObject = 0;
+    GLuint _openGLId = 0;
+};
+
 } }
 
 #endif

libraries/gpu/src/gpu/DrawTextureLinearToSRGB.slf

@@ -16,7 +16,7 @@
 <@include gpu/Color.slh@>
 
 
-LAYOUT(binding=0) uniform sampler2D colorMap;
+TEXTURE(0, sampler2D, colorMap);
 
 layout(location=0) in vec2 varTexCoord0;
 layout(location=0) out vec4 outFragColor;

libraries/gpu/src/gpu/DrawTextureSRGBToLinear.slf

@@ -16,7 +16,7 @@
 <@include gpu/Color.slh@>
 
 
-LAYOUT(binding=0) uniform sampler2D colorMap;
+TEXTURE(0, sampler2D, colorMap);
 
 layout(location=0) in vec2 varTexCoord0;
 layout(location=0) out vec4 outFragColor;