ignore axis labels when computing margins in patchwork and ggplot2

Question

I'm trying to create a scatterplot matrix using ggplot2 and patchwork.

My function for this is

library(tidyverse)
library(patchwork)
library(grid)

ggscatter <- function(df, ...) {
  plots <- df %>%
    mutate(
      across(.fns = as.character),
      .id = 1:n(),
    ) %>%
    pivot_longer(-.id, names_to = ".var", values_to = ".val") %>%
    full_join(., ., by = ".id") %>%
    nest(data = -c(.var.x, .var.y)) %>%
    mutate(
      data = pmap(., function(data, .var.x, .var.y) {
        data %>%
          mutate(
            .val.x = switch(class(df[[.var.x]]),
              "factor" = factor(.val.x),
              "numeric" = as.numeric(.val.x),
              "character" = .val.x 
            ),
            .val.y = switch(class(df[[.var.y]]),
              "factor" = factor(.val.y),
              "numeric" = as.numeric(.val.y),
              "character" = .val.y
            )
          )
      })
    ) %>%
    mutate(
      plot = pmap(., function(data, .var.x, .var.y) {
        if(.var.x == .var.y) {
          p <- grid::textGrob(.var.x, gp = gpar(fontsize = 15))
        } else {
          p <- data %>%
            ggplot(aes(x = .val.x, y = .val.y)) + 
            geom_point(
              position = position_jitter(
                width = {if(class(data$.val.x) == "factor") .2 else 0},
                height = {if(class(data$.val.y) == "factor") .2 else 0}
              )
            ) + 
            labs(
              x = .var.x,
              y = .var.y
            ) + 
            theme(
              axis.title = element_blank(), 
              axis.text.x = {if(.var.y == names(df)[length(df)] || (.var.x == names(df)[length(df)] & .var.y == names(df)[length(df) -1])) element_text() else element_blank()},
              axis.text.y = {if(.var.x == names(df)[1]  || (.var.y == names(df)[1] & .var.x == names(df)[2])) element_text() else element_blank()},
              axis.ticks.x = {if(.var.y == names(df)[length(df)] || (.var.x == names(df)[length(df)] & .var.y == names(df)[length(df) -1])) element_line() else element_blank()},
              axis.ticks.y = {if(.var.x == names(df)[1] || (.var.y == names(df)[1] & .var.x == names(df)[2])) element_line() else element_blank()},
              plot.margin = margin(0, 0, 0, 0)
            )
          
        }
        p
      })
    )
    wrap_plots(plots$plot,
      guides = "collect",
      byrow = FALSE
    )
  
}


ggscatter(iris)

This works beautifully... except for the spacing between the 1st and 2nd column and the 2nd-to-last and last rows, due to the x- and y-axis labels. Obviously, because the diagonals are textGrobs, the "simple" solution is to ignore the space the labels take up for the purpose of positioning the plots (since overflowing into this area should be okay). Changing plot.margin in theme does not help - it looks outside the space utilized by the axis text. I don't know that this is possible in patchwork, and I don't know enough about grid (or gridExtra) to see how this would be implemented. Any ideas or pointers towards a good solution?