ψ[x_, y_] = x*y*(x - a)*(y - a)

gradψ = {D[ψ[x, y], x], D[ψ[x, y], y]}

Jsq = 
      4*(Integrate[ψ[x, y], {x, 0, a}, {y, 0, a}])^2/
          Integrate[gradψ . gradψ, {x, 0, a}, {y, 0, a}]

ψ[x_, y_] = x*y/a^2 - (x + y)*x*y/a^3

gradψ = {D[ψ[x, y], x], D[ψ[x, y], y]}

Jtri = 
      4*(Integrate[ψ[x, y], {x, 0, a}, {y, 0, a - x}])^2/
          Integrate[
            gradψ . gradψ, {x, 0, a}, {y, 0, a - x}]

N[Jtri /. a -> 1]

N[Jsq /. a -> 1]