Rework postprocessor handling in GLMakie

.github/workflows/ci.yml

@@ -6,6 +6,7 @@ on:
       - '*.md'
     branches:
       - master
+      - ff/safe-context
   push:
     tags:
       - '*'

.github/workflows/compilation-benchmark.yaml

@@ -6,6 +6,7 @@ on:
       - '*.md'
     branches:
       - master
+      - ff/safe-context
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}

.github/workflows/reference_tests.yml

@@ -6,6 +6,7 @@ on:
       - '*.md'
     branches:
       - master
+      - ff/safe-context
   push:
     tags:
       - '*'

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## [Unreleased]
 
 - Fixed indexing error edge case in violin median code [#4682](https://github.com/MakieOrg/Makie.jl/pull/4682)
+- Refactored rendering in GLMakie to go through a series of steps abstracted by a render pipeline. This allows rendering to be adjusted from outside and should simplify introducing more post-processing options in the future. [#4689](https://github.com/MakieOrg/Makie.jl/pull/4689)
 
 ## [0.22.0] - 2024-12-12
 
@@ -28,6 +29,7 @@
 - Fixed gaps in corners of `poly(Rect2(...))` stroke [#4664](https://github.com/MakieOrg/Makie.jl/pull/4664)
 - Fixed an issue where `reinterpret`ed arrays of line points were not handled correctly in CairoMakie [#4668](https://github.com/MakieOrg/Makie.jl/pull/4668).
 - Fixed various issues with `markerspace = :data`, `transform_marker = true` and `rotation` for scatter in CairoMakie (incorrect marker transformations, ignored transformations, Cairo state corruption) [#4663](https://github.com/MakieOrg/Makie.jl/pull/4663)
+- Refactored OpenGL cleanup to run immediately rather than on GC [#4699](https://github.com/MakieOrg/Makie.jl/pull/4699)
 
 ## [0.21.18] - 2024-12-12
 

GLMakie/assets/shader/fragment_output.frag

@@ -33,7 +33,7 @@ void write2framebuffer(vec4 color, uvec2 id){
 
     // // if transparency == false && ssao = true
     // fragment_color = color;
-    // fragment_position = o_view_pos;
+    // fragment_position.xyz = o_view_pos;
     // fragment_normal_occlusion.xyz = o_view_normal;
 
     // // else

GLMakie/assets/shader/postprocessing/OIT_blend.frag

@@ -6,16 +6,16 @@
 in vec2 frag_uv;
 
 // contains sum_i C_i * weight(depth_i, alpha_i)
-uniform sampler2D sum_color;
+uniform sampler2D color_sum_buffer;
 // contains pod_i (1 - alpha_i)
-uniform sampler2D prod_alpha;
+uniform sampler2D transmittance_buffer;
 
 out vec4 fragment_color;
 
 void main(void)
 {
-    vec4 summed_color_weight = texture(sum_color, frag_uv);
-    float transmittance = texture(prod_alpha, frag_uv).r;
+    vec4 summed_color_weight = texture(color_sum_buffer, frag_uv);
+    float transmittance = texture(transmittance_buffer, frag_uv).r;
 
     vec3 weighted_transparent = summed_color_weight.rgb / max(summed_color_weight.a, 0.00001);
     vec3 full_weighted_transparent = weighted_transparent * (1 - transmittance);

GLMakie/assets/shader/postprocessing/SSAO.frag

@@ -2,7 +2,7 @@
 
 // SSAO
 uniform sampler2D position_buffer;
-uniform sampler2D normal_occlusion_buffer;
+uniform sampler2D normal_buffer;
 uniform sampler2D noise;
 uniform vec3 kernel[{{N_samples}}];
 uniform vec2 noise_scale;
@@ -16,18 +16,18 @@ uniform float radius;
 
 in vec2 frag_uv;
 // occlusion.xyz is a normal vector, occlusion.w the occlusion value
-out vec4 o_normal_occlusion;
+out float o_occlusion;
 
 
 void main(void)
 {
     vec3 view_pos = texture(position_buffer, frag_uv).xyz;
-    vec3 normal  = texture(normal_occlusion_buffer, frag_uv).xyz;
+    vec3 normal  = texture(normal_buffer, frag_uv).xyz;
 
     // The normal buffer gets cleared every frame. (also position, color etc)
     // If normal == vec3(1) then there is no geometry at this fragment.
     // Therefore skip SSAO calculation
-    if (normal != vec3(1)) {
+    if (normal != vec3(0)) {
         vec3 rand_vec = vec3(texture(noise, frag_uv * noise_scale).xy, 0.0);
         vec3 tangent = normalize(rand_vec - normal * dot(rand_vec, normal));
         vec3 bitangent = cross(normal, tangent);
@@ -78,13 +78,14 @@ void main(void)
             sample_frag_pos.xyz = sample_frag_pos.xyz * 0.5 + 0.5;
             sample_frag_pos.xy += (frag_uv - 0.5) * clip_pos_w / sample_clip_pos_w;
 
+            if (texture(normal_buffer, sample_frag_pos.xy).xyz == vec3(0)) continue;
 
             float sample_depth = texture(position_buffer, sample_frag_pos.xy).z;
             float range_check = smoothstep(0.0, 1.0, radius / abs(view_pos.z - sample_depth));
             occlusion += (sample_depth >= sample_view_offset.z + view_pos.z + bias ? 1.0 : 0.0) * range_check;
         }
-        o_normal_occlusion.w = occlusion / {{N_samples}};
+        o_occlusion = occlusion / {{N_samples}};
     } else {
-        o_normal_occlusion.w = 0.0;
+        o_occlusion = 0.0;
     }
 }

GLMakie/assets/shader/postprocessing/SSAO_blur.frag

@@ -1,9 +1,9 @@
 {{GLSL_VERSION}}
 
 // occlusion.w is the occlusion value
-uniform sampler2D normal_occlusion;
-uniform sampler2D color_texture;
-uniform usampler2D ids;
+uniform sampler2D occlusion_buffer;
+uniform sampler2D color_buffer;
+uniform usampler2D objectid_buffer;
 uniform vec2 inv_texel_size;
 // Settings/Attributes
 uniform int blur_range;
@@ -14,31 +14,32 @@ out vec4 fragment_color;
 void main(void)
 {
     // occlusion blur
-    uvec2 id0 = texture(ids, frag_uv).xy;
+    uvec2 id0 = texture(objectid_buffer, frag_uv).xy;
     if (id0.x == uint(0)){
-        fragment_color = texture(color_texture, frag_uv);
+        fragment_color = texture(color_buffer, frag_uv);
         // fragment_color = vec4(1,0,1,1); // show discarded
         return;
     }
 
     float blurred_occlusion = 0.0;
     float weight = 0;
 
+    // TODO: Could use :linear filtering and/ro mipmap to simplify this maybe?
     for (int x = -blur_range; x <= blur_range; ++x){
         for (int y = -blur_range; y <= blur_range; ++y){
             vec2 offset = vec2(float(x), float(y)) * inv_texel_size;
             // The id check makes it so that the blur acts per object.
             // Without this, a high (low) occlusion from one object can bleed
             // into the low (high) occlusion of another, giving an unwanted
             // shine effect.
-            uvec2 id = texture(ids, frag_uv + offset).xy;
+            uvec2 id = texture(objectid_buffer, frag_uv + offset).xy;
             float valid = float(id == id0);
-            blurred_occlusion += valid * texture(normal_occlusion, frag_uv + offset).w;
+            blurred_occlusion += valid * texture(occlusion_buffer, frag_uv + offset).x;
             weight += valid;
         }
     }
     blurred_occlusion = 1.0 - blurred_occlusion / weight;
-    fragment_color = texture(color_texture, frag_uv) * blurred_occlusion;
+    fragment_color = texture(color_buffer, frag_uv) * blurred_occlusion;
     // Display occlusion instead:
     // fragment_color = vec4(vec3(blurred_occlusion), 1.0);
 }

GLMakie/assets/shader/postprocessing/copy.frag

@@ -1,12 +1,12 @@
 {{GLSL_VERSION}}
 
-uniform sampler2D color_texture;
+uniform sampler2D color_buffer;
 in vec2 frag_uv;
 out vec4 fragment_color;
 
 void main(void)
 {
-    vec4 color = texture(color_texture, frag_uv);
+    vec4 color = texture(color_buffer, frag_uv);
     fragment_color.rgb = color.rgb;
     fragment_color.a = 1.0;
 }

GLMakie/assets/shader/postprocessing/fxaa.frag

@@ -1019,7 +1019,7 @@ FxaaFloat4 FxaaPixelShader(
 
 precision highp float;
 
-uniform sampler2D color_texture;
+uniform sampler2D color_luma_buffer;
 uniform vec2 RCPFrame;
 in vec2 frag_uv;
 
@@ -1028,23 +1028,24 @@ out vec4 fragment_color;
 
 void main(void)
 {
-    // fragment_color = texture(color_texture, frag_uv);
+    // fragment_color = texture(color_luma_buffer, frag_uv);
+    // fragment_color.rgb = vec3(texture(color_luma_buffer, frag_uv).a);
     fragment_color.rgb = FxaaPixelShader(
         frag_uv,
-        FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f),        // FxaaFloat4 fxaaConsolePosPos,
-        color_texture,                            // FxaaTex tex,
-        color_texture,                            // FxaaTex fxaaConsole360TexExpBiasNegOne,
-        color_texture,                            // FxaaTex fxaaConsole360TexExpBiasNegTwo,
-        RCPFrame,                            // FxaaFloat2 fxaaQualityRcpFrame,
-        FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f),        // FxaaFloat4 fxaaConsoleRcpFrameOpt,
-        FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f),        // FxaaFloat4 fxaaConsoleRcpFrameOpt2,
-        FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f),        // FxaaFloat4 fxaaConsole360RcpFrameOpt2,
+        FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f),       // FxaaFloat4 fxaaConsolePosPos,
+        color_luma_buffer,                        // FxaaTex tex,
+        color_luma_buffer,                        // FxaaTex fxaaConsole360TexExpBiasNegOne,
+        color_luma_buffer,                        // FxaaTex fxaaConsole360TexExpBiasNegTwo,
+        RCPFrame,                                 // FxaaFloat2 fxaaQualityRcpFrame,
+        FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f),       // FxaaFloat4 fxaaConsoleRcpFrameOpt,
+        FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f),       // FxaaFloat4 fxaaConsoleRcpFrameOpt2,
+        FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f),       // FxaaFloat4 fxaaConsole360RcpFrameOpt2,
         0.75f,                                    // FxaaFloat fxaaQualitySubpix,
-        0.166f,                                    // FxaaFloat fxaaQualityEdgeThreshold,
-        0.0833f,                                // FxaaFloat fxaaQualityEdgeThresholdMin,
-        0.0f,                                    // FxaaFloat fxaaConsoleEdgeSharpness,
-        0.0f,                                    // FxaaFloat fxaaConsoleEdgeThreshold,
-        0.0f,                                    // FxaaFloat fxaaConsoleEdgeThresholdMin,
+        0.166f,                                   // FxaaFloat fxaaQualityEdgeThreshold,
+        0.0833f,                                  // FxaaFloat fxaaQualityEdgeThresholdMin,
+        0.0f,                                     // FxaaFloat fxaaConsoleEdgeSharpness,
+        0.0f,                                     // FxaaFloat fxaaConsoleEdgeThreshold,
+        0.0f,                                     // FxaaFloat fxaaConsoleEdgeThresholdMin,
         FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f)        // FxaaFloat fxaaConsole360ConstDir,
     ).rgb;
 }

GLMakie/assets/shader/postprocessing/postprocess.frag

@@ -1,9 +1,11 @@
 {{GLSL_VERSION}}
 
+{{FILTER_IN_SHADER}}
+
 in vec2 frag_uv;
 
-uniform sampler2D color_texture;
-uniform usampler2D object_ids;
+uniform sampler2D color_buffer;
+uniform usampler2D objectid_buffer;
 
 layout(location=0) out vec4 fragment_color;
 
@@ -21,20 +23,25 @@ bool unpack_bool(uint id) {
 
 void main(void)
 {
-    vec4 color = texture(color_texture, frag_uv).rgba;
+    vec4 color = texture(color_buffer, frag_uv).rgba;
     if(color.a <= 0){
         discard;
     }
 
-    uint id = texture(object_ids, frag_uv).x;
+    uint id = texture(objectid_buffer, frag_uv).x;
     // do tonemappings
     //opaque = linear_tone_mapping(color.rgb, 1.8);  // linear color output
     fragment_color.rgb = color.rgb;
+
+#ifdef FILTER_IN_SHADER
     // we store fxaa = true/false in highbit of the object id
     if (unpack_bool(id)) {
         fragment_color.a = dot(color.rgb, vec3(0.299, 0.587, 0.114)); // compute luma
     } else {
         // we disable fxaa by setting luma to 1
         fragment_color.a = 1.0;
     }
+#else
+    fragment_color.a = dot(color.rgb, vec3(0.299, 0.587, 0.114)); // compute luma
+#endif
 }

GLMakie/src/GLAbstraction/AbstractGPUArray.jl

@@ -190,8 +190,8 @@ gpu_setindex!(t) = error("gpu_setindex! not implemented for: $(typeof(t)). This
 max_dim(t)       = error("max_dim not implemented for: $(typeof(t)). This happens, when you call setindex! on an array, without implementing the GPUArray interface")
 
 
-function (::Type{GPUArrayType})(data::Observable; kw...) where GPUArrayType <: GPUArray
-    gpu_mem = GPUArrayType(data[]; kw...)
+function (::Type{GPUArrayType})(context, data::Observable; kw...) where GPUArrayType <: GPUArray
+    gpu_mem = GPUArrayType(context, data[]; kw...)
     # TODO merge these and handle update tracking during construction
     obs2 = on(new_data -> update!(gpu_mem, new_data), data)
     if GPUArrayType <: TextureBuffer

GLMakie/src/GLAbstraction/GLAbstraction.jl

@@ -18,6 +18,20 @@ import FixedPointNumbers: N0f8, N0f16, N0f8, Normed
 
 import Base: merge, resize!, similar, length, getindex, setindex!
 
+# Debug tools
+const GLMAKIE_DEBUG = Ref(false)
+
+# implemented in GLMakie/glwindow.jl
+function require_context_no_error(args...) end
+
+function require_context(ctx, current = ShaderAbstractions.current_context())
+    msg = require_context_no_error(ctx, current)
+    isnothing(msg) && return nothing
+    error(msg)
+end
+
+export require_context
+
 include("AbstractGPUArray.jl")
 
 #Methods which get overloaded by GLExtendedFunctions.jl:
@@ -41,9 +55,9 @@ import ModernGL.glGetShaderiv
 import ModernGL.glViewport
 import ModernGL.glScissor
 
+include("shaderabstraction.jl")
 include("GLUtils.jl")
 
-include("shaderabstraction.jl")
 include("GLTypes.jl")
 export GLProgram                # Shader/program object
 export Texture                  # Texture object, basically a 1/2/3D OpenGL data array
@@ -99,4 +113,12 @@ export getUniformsInfo
 export getProgramInfo
 export getAttributesInfo
 
+include("GLFramebuffer.jl")
+export GLRenderbuffer
+export GLFramebuffer
+export attach_colorbuffer, attach_depthbuffer, attach_stencilbuffer, attach_depthstencilbuffer
+export get_attachment, get_buffer
+export set_draw_buffers
+export check_framebuffer
+
 end # module