Memory grows during PipeWire screencasts on NVIDIA (~100–440 MB/h): per-texture read fences in smithay's GlesRenderer leak inside the NVIDIA driver

// Minimal repro: per-frame glFenceSync/glWaitSync/glDeleteSync around sampling a dmabuf-imported
// texture, rendering into a dmabuf-imported texture FBO, on an EGL GBM-platform surfaceless context.
// This is what smithay's GlesRenderer did per texture per frame (TextureSync::update_read).
//   cc repro.c -o repro -lEGL -lGLESv2 -lgbm
//   ./repro [fences 0|1|2 (2: no glWaitSync)] [frames] [new_fbo_each_frame 0|1]
#define EGL_EGLEXT_PROTOTYPES
#define GL_GLEXT_PROTOTYPES
#include <EGL/egl.h>
#include <EGL/eglext.h>
#include <GLES3/gl3.h>
#include <GLES2/gl2ext.h>
#include <gbm.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

static long rss_kb(void) {
    FILE *f = fopen("/proc/self/status", "r");
    char line[256];
    long kb = -1;
    while (fgets(line, sizeof line, f))
        if (!strncmp(line, "VmRSS:", 6)) kb = atol(line + 6);
    fclose(f);
    return kb;
}

static PFNEGLCREATEIMAGEKHRPROC create_image;
static PFNGLEGLIMAGETARGETTEXTURE2DOESPROC image_target_texture;

static GLuint dmabuf_texture(EGLDisplay dpy, struct gbm_device *gbm, int w, int h) {
    struct gbm_bo *bo = gbm_bo_create(gbm, w, h, GBM_FORMAT_ARGB8888, GBM_BO_USE_RENDERING);
    if (!bo) { fprintf(stderr, "gbm_bo_create failed\n"); exit(1); }
    EGLint attrs[] = {
        EGL_WIDTH, w, EGL_HEIGHT, h, EGL_LINUX_DRM_FOURCC_EXT, GBM_FORMAT_ARGB8888,
        EGL_DMA_BUF_PLANE0_FD_EXT, gbm_bo_get_fd(bo),
        EGL_DMA_BUF_PLANE0_OFFSET_EXT, (EGLint)gbm_bo_get_offset(bo, 0),
        EGL_DMA_BUF_PLANE0_PITCH_EXT, (EGLint)gbm_bo_get_stride(bo),
        EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT, (EGLint)(gbm_bo_get_modifier(bo) & 0xffffffff),
        EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT, (EGLint)(gbm_bo_get_modifier(bo) >> 32),
        EGL_NONE};
    EGLImageKHR img = create_image(dpy, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, NULL, attrs);
    if (img == EGL_NO_IMAGE_KHR) { fprintf(stderr, "eglCreateImageKHR failed 0x%x\n", eglGetError()); exit(1); }
    GLuint tex;
    glGenTextures(1, &tex);
    glBindTexture(GL_TEXTURE_2D, tex);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    image_target_texture(GL_TEXTURE_2D, img);
    return tex;
}

static GLuint fbo_for(GLuint tex) {
    GLuint fbo;
    glGenFramebuffers(1, &fbo);
    glBindFramebuffer(GL_FRAMEBUFFER, fbo);
    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, tex, 0);
    if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) { fprintf(stderr, "fbo incomplete\n"); exit(1); }
    return fbo;
}

int main(int argc, char **argv) {
    int fences = argc > 1 ? atoi(argv[1]) : 1;
    long frames = argc > 2 ? atol(argv[2]) : 200000;
    int new_fbo = argc > 3 ? atoi(argv[3]) : 0;
    int fd = open("/dev/dri/renderD128", O_RDWR | O_CLOEXEC);
    struct gbm_device *gbm = gbm_create_device(fd);
    EGLDisplay dpy = eglGetPlatformDisplay(EGL_PLATFORM_GBM_KHR, gbm, NULL);
    eglInitialize(dpy, NULL, NULL);
    eglBindAPI(EGL_OPENGL_ES_API);
    EGLint cattrs[] = {EGL_CONTEXT_CLIENT_VERSION, 3, EGL_NONE};
    EGLContext ctx = eglCreateContext(dpy, EGL_NO_CONFIG_KHR, EGL_NO_CONTEXT, cattrs);
    if (!eglMakeCurrent(dpy, EGL_NO_SURFACE, EGL_NO_SURFACE, ctx)) { fprintf(stderr, "make current failed\n"); return 1; }
    create_image = (PFNEGLCREATEIMAGEKHRPROC)eglGetProcAddress("eglCreateImageKHR");
    image_target_texture = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglGetProcAddress("glEGLImageTargetTexture2DOES");
    printf("%s / %s\n", glGetString(GL_RENDERER), glGetString(GL_VERSION));

    enum { W = 1920, H = 1080, NSRC = 8 };   // a screen-sized target, a few "window" textures
    GLuint dst = dmabuf_texture(dpy, gbm, W, H), src[NSRC];
    for (int i = 0; i < NSRC; i++) src[i] = dmabuf_texture(dpy, gbm, 800, 600);
    GLuint fbo = fbo_for(dst);

    const char *vs = "#version 300 es\nin vec2 p; out vec2 uv; void main(){uv=p*0.5+0.5; gl_Position=vec4(p,0,1);}";
    const char *fs = "#version 300 es\nprecision mediump float; in vec2 uv; uniform sampler2D t; out vec4 c; void main(){c=texture(t,uv);}";
    GLuint prog = glCreateProgram(), v = glCreateShader(GL_VERTEX_SHADER), f = glCreateShader(GL_FRAGMENT_SHADER);
    glShaderSource(v, 1, &vs, NULL); glCompileShader(v);
    glShaderSource(f, 1, &fs, NULL); glCompileShader(f);
    glAttachShader(prog, v); glAttachShader(prog, f); glLinkProgram(prog); glUseProgram(prog);
    static const float quad[] = {-1, -1, 1, -1, -1, 1, 1, 1};
    GLuint vbo;
    glGenBuffers(1, &vbo); glBindBuffer(GL_ARRAY_BUFFER, vbo);
    glBufferData(GL_ARRAY_BUFFER, sizeof quad, quad, GL_STATIC_DRAW);
    glEnableVertexAttribArray(0); glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, 0);

    GLsync read_sync[NSRC] = {0};
    long start = rss_kb();
    printf("fences=%d new_fbo_each_frame=%d frames=%ld rss_start=%ld MB\n", fences, new_fbo, frames, start / 1024);
    for (long n = 1; n <= frames; n++) {
        GLuint target = new_fbo ? fbo_for(dst) : fbo;
        glBindFramebuffer(GL_FRAMEBUFFER, target);
        glViewport(0, 0, W, H);
        for (int i = 0; i < NSRC; i++) {
            glBindTexture(GL_TEXTURE_2D, src[i]);
            glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
            if (fences) {   // smithay TextureSync::update_read
                if (read_sync[i]) {
                    if (fences == 1) glWaitSync(read_sync[i], 0, GL_TIMEOUT_IGNORED);   // 2: no server wait
                    glDeleteSync(read_sync[i]);
                }
                read_sync[i] = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
            }
        }
        GLsync done = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);   // the frame's sync point
        glFlush();
        glClientWaitSync(done, GL_SYNC_FLUSH_COMMANDS_BIT, 1000000000);
        glDeleteSync(done);
        if (new_fbo) { glBindFramebuffer(GL_FRAMEBUFFER, 0); glDeleteFramebuffers(1, &target); }
        if (n % (frames / 10) == 0)
            printf("frame %7ld  rss %6ld MB  (+%ld MB)\n", n, rss_kb() / 1024, (rss_kb() - start) / 1024), fflush(stdout);
    }
    return 0;
}