Render interface: Make layer compositing a separate function instead …

…of being a part of layer pop

The purpose of each function should now be more clear. With this change, we can now composite two arbitrary layers which simplifies some use cases of compositing.

This commit also uses the changed interface to fix some artifacts when rendering back-drop filter blur combined with plain filter blur.

Backends/RmlUi_Renderer_GL3.cpp

@@ -1736,9 +1736,9 @@ void RenderInterface_GL3::ReleaseShader(Rml::CompiledShaderHandle shader_handle)
 	delete reinterpret_cast<CompiledShader*>(shader_handle);
 }
 
-void RenderInterface_GL3::BlitTopLayerToPostprocessPrimary()
+void RenderInterface_GL3::BlitLayerToPostprocessPrimary(Rml::LayerHandle layer_handle)
 {
-	const Gfx::FramebufferData& source = render_layers.GetTopLayer();
+	const Gfx::FramebufferData& source = render_layers.GetLayer(layer_handle);
 	const Gfx::FramebufferData& destination = render_layers.GetPostprocessPrimary();
 	glBindFramebuffer(GL_READ_FRAMEBUFFER, source.framebuffer);
 	glBindFramebuffer(GL_DRAW_FRAMEBUFFER, destination.framebuffer);
@@ -1873,57 +1873,31 @@ void RenderInterface_GL3::RenderFilters(Rml::Span<const Rml::CompiledFilterHandl
 	Gfx::CheckGLError("RenderFilter");
 }
 
-void RenderInterface_GL3::PushLayer(Rml::LayerFill layer_fill)
+Rml::LayerHandle RenderInterface_GL3::PushLayer()
 {
-	const Gfx::FramebufferData source = render_layers.GetTopLayer();
+	const Rml::LayerHandle layer_handle = render_layers.PushLayer();
 
-	if (layer_fill == Rml::LayerFill::Link)
-		render_layers.PushLayerClone();
-	else
-		render_layers.PushLayer();
+	glBindFramebuffer(GL_FRAMEBUFFER, render_layers.GetLayer(layer_handle).framebuffer);
+	glClear(GL_COLOR_BUFFER_BIT);
 
-	if (layer_fill == Rml::LayerFill::Copy)
-	{
-		const Gfx::FramebufferData& destination = render_layers.GetTopLayer();
-		glBindFramebuffer(GL_READ_FRAMEBUFFER, source.framebuffer);
-		glBindFramebuffer(GL_DRAW_FRAMEBUFFER, destination.framebuffer);
-		// Note that the blit region will be clipped by any active scissor region.
-		glBlitFramebuffer(0, 0, source.width, source.height, 0, 0, destination.width, destination.height, GL_COLOR_BUFFER_BIT, GL_NEAREST);
-		glBindFramebuffer(GL_FRAMEBUFFER, destination.framebuffer);
-	}
-	else
-	{
-		glBindFramebuffer(GL_FRAMEBUFFER, render_layers.GetTopLayer().framebuffer);
-		if (layer_fill == Rml::LayerFill::Clear)
-			glClear(GL_COLOR_BUFFER_BIT);
-	}
+	return layer_handle;
 }
 
-void RenderInterface_GL3::PopLayer(Rml::BlendMode blend_mode, Rml::Span<const Rml::CompiledFilterHandle> filters)
+void RenderInterface_GL3::CompositeLayers(Rml::LayerHandle source_handle, Rml::LayerHandle destination_handle, Rml::BlendMode blend_mode,
+	Rml::Span<const Rml::CompiledFilterHandle> filters)
 {
 	using Rml::BlendMode;
 
-	if (blend_mode == BlendMode::Discard)
-	{
-		RMLUI_ASSERT(filters.empty());
-		render_layers.PopLayer();
-		glBindFramebuffer(GL_FRAMEBUFFER, render_layers.GetTopLayer().framebuffer);
-		return;
-	}
-
-	// Blit stack to filter rendering buffer. Do this regardless of whether we actually have any filters to be applied,
-	// because we need to resolve the multi-sampled framebuffer in any case.
+	// Blit source layer to postprocessing buffer. Do this regardless of whether we actually have any filters to be
+	// applied, because we need to resolve the multi-sampled framebuffer in any case.
 	// @performance If we have BlendMode::Replace and no filters or mask then we can just blit directly to the destination.
-	BlitTopLayerToPostprocessPrimary();
+	BlitLayerToPostprocessPrimary(source_handle);
 
 	// Render the filters, the PostprocessPrimary framebuffer is used for both input and output.
 	RenderFilters(filters);
 
-	// Pop the active layer, thereby activating the beneath layer.
-	render_layers.PopLayer();
-
-	// Render to the activated layer. Apply any mask if active.
-	glBindFramebuffer(GL_FRAMEBUFFER, render_layers.GetTopLayer().framebuffer);
+	// Render to the destination layer.
+	glBindFramebuffer(GL_FRAMEBUFFER, render_layers.GetLayer(destination_handle).framebuffer);
 	Gfx::BindTexture(render_layers.GetPostprocessPrimary());
 
 	UseProgram(ProgramId::Passthrough);
@@ -1936,7 +1910,16 @@ void RenderInterface_GL3::PopLayer(Rml::BlendMode blend_mode, Rml::Span<const Rm
 	if (blend_mode == BlendMode::Replace)
 		glEnable(GL_BLEND);
 
-	Gfx::CheckGLError("PopLayer");
+	if (destination_handle != render_layers.GetTopLayerHandle())
+		glBindFramebuffer(GL_FRAMEBUFFER, render_layers.GetTopLayer().framebuffer);
+
+	Gfx::CheckGLError("CompositeLayers");
+}
+
+void RenderInterface_GL3::PopLayer()
+{
+	render_layers.PopLayer();
+	glBindFramebuffer(GL_FRAMEBUFFER, render_layers.GetTopLayer().framebuffer);
 }
 
 Rml::TextureHandle RenderInterface_GL3::SaveLayerAsTexture(Rml::Vector2i dimensions)
@@ -1945,7 +1928,7 @@ Rml::TextureHandle RenderInterface_GL3::SaveLayerAsTexture(Rml::Vector2i dimensi
 	if (!render_texture)
 		return {};
 
-	BlitTopLayerToPostprocessPrimary();
+	BlitLayerToPostprocessPrimary(render_layers.GetTopLayerHandle());
 
 	RMLUI_ASSERT(scissor_state.Valid() && render_texture);
 	const Rml::Rectanglei initial_scissor_state = scissor_state;
@@ -1982,7 +1965,7 @@ Rml::TextureHandle RenderInterface_GL3::SaveLayerAsTexture(Rml::Vector2i dimensi
 
 Rml::CompiledFilterHandle RenderInterface_GL3::SaveLayerAsMaskImage()
 {
-	BlitTopLayerToPostprocessPrimary();
+	BlitLayerToPostprocessPrimary(render_layers.GetTopLayerHandle());
 
 	const Gfx::FramebufferData& source = render_layers.GetPostprocessPrimary();
 	const Gfx::FramebufferData& destination = render_layers.GetBlendMask();
@@ -2045,7 +2028,7 @@ RenderInterface_GL3::RenderLayerStack::~RenderLayerStack()
 	DestroyFramebuffers();
 }
 
-void RenderInterface_GL3::RenderLayerStack::PushLayer()
+Rml::LayerHandle RenderInterface_GL3::RenderLayerStack::PushLayer()
 {
 	RMLUI_ASSERT(layers_size <= (int)fb_layers.size());
 
@@ -2059,29 +2042,30 @@ void RenderInterface_GL3::RenderLayerStack::PushLayer()
 	}
 
 	layers_size += 1;
-}
-
-void RenderInterface_GL3::RenderLayerStack::PushLayerClone()
-{
-	RMLUI_ASSERT(layers_size > 0);
-	fb_layers.insert(fb_layers.begin() + layers_size, Gfx::FramebufferData{fb_layers[layers_size - 1]});
-	layers_size += 1;
+	return GetTopLayerHandle();
 }
 
 void RenderInterface_GL3::RenderLayerStack::PopLayer()
 {
 	RMLUI_ASSERT(layers_size > 0);
 	layers_size -= 1;
+}
 
-	// Only cloned framebuffers are removed. Other framebuffers remain for later re-use.
-	if (IsCloneOfBelow(layers_size))
-		fb_layers.erase(fb_layers.begin() + layers_size);
+const Gfx::FramebufferData& RenderInterface_GL3::RenderLayerStack::GetLayer(Rml::LayerHandle layer) const
+{
+	RMLUI_ASSERT((size_t)layer < (size_t)layers_size);
+	return fb_layers[layer];
 }
 
 const Gfx::FramebufferData& RenderInterface_GL3::RenderLayerStack::GetTopLayer() const
+{
+	return GetLayer(GetTopLayerHandle());
+}
+
+Rml::LayerHandle RenderInterface_GL3::RenderLayerStack::GetTopLayerHandle() const
 {
 	RMLUI_ASSERT(layers_size > 0);
-	return fb_layers[layers_size - 1];
+	return static_cast<Rml::LayerHandle>(layers_size - 1);
 }
 
 void RenderInterface_GL3::RenderLayerStack::SwapPostprocessPrimarySecondary()
@@ -2123,13 +2107,6 @@ void RenderInterface_GL3::RenderLayerStack::DestroyFramebuffers()
 		Gfx::DestroyFramebuffer(fb);
 }
 
-bool RenderInterface_GL3::RenderLayerStack::IsCloneOfBelow(int layer_index) const
-{
-	const bool result =
-		(layer_index >= 1 && layer_index < (int)fb_layers.size() && fb_layers[layer_index].framebuffer == fb_layers[layer_index - 1].framebuffer);
-	return result;
-}
-
 const Gfx::FramebufferData& RenderInterface_GL3::RenderLayerStack::EnsureFramebufferPostprocess(int index)
 {
 	RMLUI_ASSERT(index < (int)fb_postprocess.size())

Backends/RmlUi_Renderer_GL3.h

@@ -78,8 +78,10 @@ class RenderInterface_GL3 : public Rml::RenderInterface {
 
 	void SetTransform(const Rml::Matrix4f* transform) override;
 
-	void PushLayer(Rml::LayerFill layer_fill) override;
-	void PopLayer(Rml::BlendMode blend_mode, Rml::Span<const Rml::CompiledFilterHandle> filters) override;
+	Rml::LayerHandle PushLayer() override;
+	void CompositeLayers(Rml::LayerHandle source, Rml::LayerHandle destination, Rml::BlendMode blend_mode,
+		Rml::Span<const Rml::CompiledFilterHandle> filters) override;
+	void PopLayer() override;
 
 	Rml::TextureHandle SaveLayerAsTexture(Rml::Vector2i dimensions) override;
 
@@ -103,7 +105,7 @@ class RenderInterface_GL3 : public Rml::RenderInterface {
 	int GetUniformLocation(UniformId uniform_id) const;
 	void SubmitTransformUniform(Rml::Vector2f translation);
 
-	void BlitTopLayerToPostprocessPrimary();
+	void BlitLayerToPostprocessPrimary(Rml::LayerHandle layer_handle);
 	void RenderFilters(Rml::Span<const Rml::CompiledFilterHandle> filter_handles);
 
 	void SetScissor(Rml::Rectanglei region, bool vertically_flip = false);
@@ -144,15 +146,14 @@ class RenderInterface_GL3 : public Rml::RenderInterface {
 		~RenderLayerStack();
 
 		// Push a new layer. All references to previously retrieved layers are invalidated.
-		void PushLayer();
-
-		// Push a clone of the active layer. All references to previously retrieved layers are invalidated.
-		void PushLayerClone();
+		Rml::LayerHandle PushLayer();
 
 		// Pop the top layer. All references to previously retrieved layers are invalidated.
 		void PopLayer();
 
+		const Gfx::FramebufferData& GetLayer(Rml::LayerHandle layer) const;
 		const Gfx::FramebufferData& GetTopLayer() const;
+		Rml::LayerHandle GetTopLayerHandle() const;
 
 		const Gfx::FramebufferData& GetPostprocessPrimary() { return EnsureFramebufferPostprocess(0); }
 		const Gfx::FramebufferData& GetPostprocessSecondary() { return EnsureFramebufferPostprocess(1); }
@@ -166,7 +167,6 @@ class RenderInterface_GL3 : public Rml::RenderInterface {
 
 	private:
 		void DestroyFramebuffers();
-		bool IsCloneOfBelow(int layer_index) const;
 		const Gfx::FramebufferData& EnsureFramebufferPostprocess(int index);
 
 		int width = 0, height = 0;

Include/RmlUi/Core/RenderInterface.h

@@ -41,16 +41,9 @@ enum class ClipMaskOperation {
 	SetInverse, // Set the clip mask to the area *outside* the rendered geometry, clearing any existing clip mask.
 	Intersect,  // Intersect the clip mask with the area of the rendered geometry.
 };
-enum class LayerFill {
-	None,  // No operation necessary, does not care about the layer color.
-	Clear, // Clear the layer to transparent black.
-	Copy,  // Copy the color data from the previous layer.
-	Link,  // Link the color data with the previous layer.
-};
 enum class BlendMode {
 	Blend,   // Normal alpha blending.
 	Replace, // Replace the destination colors from the source.
-	Discard, // Leave the destination colors unaltered.
 };
 
 /**
@@ -119,7 +112,8 @@ class RMLUICORE_API RenderInterface : public NonCopyMoveable {
 	/// @param[in] geometry The compiled geometry to render.
 	/// @param[in] translation The translation to apply to the geometry.
 	/// @note When enabled, the clip mask should hide any rendered contents outside the area of the mask.
-	/// @note The clip mask applies to all other functions that render with a geometry handle, and only those.
+	/// @note The clip mask applies exclusively to all other functions that render with a geometry handle, in addition
+	/// to the layer compositing function while rendering to its destination.
 	virtual void RenderToClipMask(ClipMaskOperation operation, CompiledGeometryHandle geometry, Vector2f translation);
 
 	/// Called by RmlUi when it wants the renderer to use a new transform matrix.
@@ -129,21 +123,27 @@ class RMLUICORE_API RenderInterface : public NonCopyMoveable {
 	/// @note The transform applies to all functions that render with a geometry handle, and only those.
 	virtual void SetTransform(const Matrix4f* transform);
 
-	/// Called by RmlUi when it wants to push a new layer onto the render stack.
-	/// @param[in] layer_fill Specifies how the color data of the new layer should be filled.
-	virtual void PushLayer(LayerFill layer_fill);
-	/// Called by RmlUi when it wants to pop the render layer stack, after applying filters to the top layer and blending it into the layer below.
-	/// @param[in] blend_mode The mode used to blend the top layer into the one below.
-	/// @param[in] filters A list of compiled filters which should be applied to the top layer before blending.
-	virtual void PopLayer(BlendMode blend_mode, Span<const CompiledFilterHandle> filters);
+	/// Called by RmlUi when it wants to push a new layer onto the render stack, setting it as the new render target.
+	/// @return An application-specified handle representing the new layer. The value 'zero' is reserved for the initial base layer.
+	/// @note The new layer should be initialized to transparent black within the current scissor region.
+	virtual LayerHandle PushLayer();
+	/// Composite two layers with the given blend mode and apply filters.
+	/// @param[in] source The source layer.
+	/// @param[in] destination The destination layer.
+	/// @param[in] blend_mode The mode used to blend the source layer onto the destination layer.
+	/// @param[in] filters A list of compiled filters which should be applied before blending.
+	/// @note Source and destination can reference the same layer.
+	virtual void CompositeLayers(LayerHandle source, LayerHandle destination, BlendMode blend_mode, Span<const CompiledFilterHandle> filters);
+	/// Called by RmlUi when it wants to pop the render layer stack, setting the new top layer as the render target.
+	virtual void PopLayer();
 
 	/// Called by RmlUi when it wants to store the current layer as a new texture to be rendered later with geometry.
 	/// @param[in] dimensions The dimensions of the texture, to be copied from the top-left part of the viewport.
 	/// @return An application-specified handle to the new texture.
 	virtual TextureHandle SaveLayerAsTexture(Vector2i dimensions);
 
 	/// Called by RmlUi when it wants to store the current layer as a mask image, to be applied later as a filter.
-	/// @return An application-specified handle to a new filter representng the stored mask image.
+	/// @return An application-specified handle to a new filter representing the stored mask image.
 	virtual CompiledFilterHandle SaveLayerAsMaskImage();
 
 	/// Called by RmlUi when it wants to compile a new filter.

Include/RmlUi/Core/RenderManager.h

@@ -103,8 +103,12 @@ class RMLUICORE_API RenderManager : NonCopyMoveable {
 	CompiledFilter CompileFilter(const String& name, const Dictionary& parameters);
 	CompiledShader CompileShader(const String& name, const Dictionary& parameters);
 
-	void PushLayer(LayerFill layer_fill);
-	void PopLayer(BlendMode blend_mode, Span<const CompiledFilterHandle> filters);
+	LayerHandle PushLayer();
+	void CompositeLayers(LayerHandle source, LayerHandle destination, BlendMode blend_mode, Span<const CompiledFilterHandle> filters);
+	void PopLayer();
+
+	LayerHandle GetTopLayer() const;
+	LayerHandle GetNextLayer() const;
 
 	CompiledFilter SaveLayerAsMaskImage();
 
@@ -142,6 +146,8 @@ class RMLUICORE_API RenderManager : NonCopyMoveable {
 	RenderState state;
 	Vector2i viewport_dimensions;
 
+	Vector<LayerHandle> render_stack;
+
 	friend class RenderManagerAccess;
 };
 

Include/RmlUi/Core/StableVector.h

@@ -40,8 +40,9 @@ namespace Rml {
     A vector-like container that returns stable indices to refer to entries.
 
     The indices are only invalidated when the element is erased. Pointers on the other hand are invalidated just like for a
-    vector. The container is implemented as a vector with a separate bit mask to track free slots. For simplicity, freed
-    slots are simply replaced with value-initialized elements.
+    vector. The container is implemented as a vector with a separate bit mask to track free slots.
+
+    @note For simplicity, freed slots are simply replaced with value-initialized elements instead of being destroyed.
  */
 template <typename T>
 class StableVector {

Include/RmlUi/Core/Types.h

@@ -110,6 +110,7 @@ using CompiledShaderHandle = uintptr_t;
 using DecoratorDataHandle = uintptr_t;
 using FontFaceHandle = uintptr_t;
 using FontEffectsHandle = uintptr_t;
+using LayerHandle = uintptr_t;
 
 using ElementPtr = UniqueReleaserPtr<Element>;
 using ContextPtr = UniqueReleaserPtr<Context>;