drm/simpledrm: Preallocate format-conversion buffer in atomic_check

Preallocate the format-conversion state's storage in the plane's
atomic_check function if a format conversion is necessary. Allows
the update to fail if no memory is available. Avoids the same
allocation within atomic_update, which may not fail.

Also inline drm_plane_helper_atomic_check() into the driver and thus
return early for invisible planes. Avoids memory allocation entirely
in this case.

Signed-off-by: Thomas Zimmermann <tzimmermann@suse.de>
Reviewed-by: Javier Martinez Canillas <javierm@redhat.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20231009141018.11291-6-tzimmermann@suse.de

diff --git a/drivers/gpu/drm/tiny/simpledrm.c b/drivers/gpu/drm/tiny/simpledrm.c
index 4916e9b0d46a3a86982ccf5ce7ab780593a97e96..22a827102dfd08674638aba11b1f5f7d3c9784eb 100644 (file)
--- a/drivers/gpu/drm/tiny/simpledrm.c
+++ b/drivers/gpu/drm/tiny/simpledrm.c
@@ -19,6 +19,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fbdev_generic.h>
 #include <drm/drm_format_helper.h>
+#include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_gem_shmem_helper.h>
@@ -579,6 +580,44 @@ static const uint64_t simpledrm_primary_plane_format_modifiers[] = {
        DRM_FORMAT_MOD_INVALID
 };
 
+static int simpledrm_primary_plane_helper_atomic_check(struct drm_plane *plane,
+                                                      struct drm_atomic_state *state)
+{
+       struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
+       struct drm_shadow_plane_state *new_shadow_plane_state =
+               to_drm_shadow_plane_state(new_plane_state);
+       struct drm_framebuffer *new_fb = new_plane_state->fb;
+       struct drm_crtc *new_crtc = new_plane_state->crtc;
+       struct drm_crtc_state *new_crtc_state = NULL;
+       struct drm_device *dev = plane->dev;
+       struct simpledrm_device *sdev = simpledrm_device_of_dev(dev);
+       int ret;
+
+       if (new_crtc)
+               new_crtc_state = drm_atomic_get_new_crtc_state(state, new_crtc);
+
+       ret = drm_atomic_helper_check_plane_state(new_plane_state, new_crtc_state,
+                                                 DRM_PLANE_NO_SCALING,
+                                                 DRM_PLANE_NO_SCALING,
+                                                 false, false);
+       if (ret)
+               return ret;
+       else if (!new_plane_state->visible)
+               return 0;
+
+       if (new_fb->format != sdev->format) {
+               void *buf;
+
+               /* format conversion necessary; reserve buffer */
+               buf = drm_format_conv_state_reserve(&new_shadow_plane_state->fmtcnv_state,
+                                                   sdev->pitch, GFP_KERNEL);
+               if (!buf)
+                       return -ENOMEM;
+       }
+
+       return 0;
+}
+
 static void simpledrm_primary_plane_helper_atomic_update(struct drm_plane *plane,
                                                         struct drm_atomic_state *state)
 {
@@ -635,7 +674,7 @@ static void simpledrm_primary_plane_helper_atomic_disable(struct drm_plane *plan
 
 static const struct drm_plane_helper_funcs simpledrm_primary_plane_helper_funcs = {
        DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
-       .atomic_check = drm_plane_helper_atomic_check,
+       .atomic_check = simpledrm_primary_plane_helper_atomic_check,
        .atomic_update = simpledrm_primary_plane_helper_atomic_update,
        .atomic_disable = simpledrm_primary_plane_helper_atomic_disable,
 };