diff --git a/kitty/screen.c b/kitty/screen.c
index c062fe51d..8b163ba66 100644
--- a/kitty/screen.c
+++ b/kitty/screen.c
@@ -5086,11 +5086,17 @@ screen_fractional_scroll(Screen *self, double amt) {
     double pixels = fractional_part * self->cell_size.height;
     if (amt > 0) {  // downwards
         if (fractional_part != 0) pixels = MAX(1, pixels);
-        pixels = pixels > self->pixel_scroll_offset_y ? pixels - self->pixel_scroll_offset_y : 0;
-        self->pixel_scroll_offset_y = 0;
-        self->scrolled_by = self->scrolled_by > (unsigned)lines ? self->scrolled_by - lines : 0;
-        if (pixels > 0 && self->scrolled_by > 0) {
-            self->scrolled_by--; self->pixel_scroll_offset_y = self->cell_size.height - pixels;
+        if (lines > (int)self->scrolled_by) {
+            self->scrolled_by = 0; self->pixel_scroll_offset_y = 0;
+        } else {
+            self->scrolled_by -= lines;
+            if (pixels <= self->pixel_scroll_offset_y) self->pixel_scroll_offset_y -= pixels;
+            else {
+                self->pixel_scroll_offset_y = 0;
+                if (self->scrolled_by) {
+                    self->scrolled_by--; self->pixel_scroll_offset_y = self->cell_size.height - pixels;
+                }
+            }
         }
     } else {
         if (fractional_part != 0) pixels = MIN(-1, pixels);