sg: Fix culling in layers

Pick-to: 6.10 6.9 6.8
Fixes: QTBUG-136611
Change-Id: If2a0a0365ca24360d850ffce98c0bec4a3961976
Reviewed-by: Jonas Karlsson <jonas.karlsson@qt.io>
This commit is contained in:
Laszlo Agocs 2025-07-21 11:04:09 +02:00
parent 8effdd97d9
commit e32d335c8f
11 changed files with 568 additions and 7 deletions

View File

@ -130,10 +130,13 @@ void CustomRenderNode::prepare()
if (!m_pipeline) {
m_pipeline.reset(rhi->newGraphicsPipeline());
// If layer.enabled == true on our QQuickItem, the rendering face is flipped for
// backends with isYUpInFrameBuffer == true (OpenGL). This does not happen with
// RHI backends with isYUpInFrameBuffer == false. We swap the triangle winding
// order to work around this.
// If layer.enabled == true on our QQuickItem and layer.textureMirroring is the
// default MirrorVertically, the winding order needs to be inverted when
// isYUpInFrameBuffer == true (OpenGL). This does not happen with other backends,
// unless textureMirroring is changed so that the situation is inverted, meaning
// GL needs no adjustments while others need the opposite winding order. Here we
// choose the replicate, to a degree, what the scenegraph renderer does, but note that
// this is only correct as long as textureMirroring is not changed from the default.
m_pipeline->setFrontFace(renderTarget()->resourceType() == QRhiResource::TextureRenderTarget
&& rhi->isYUpInFramebuffer()
? QRhiGraphicsPipeline::CW

View File

@ -48,6 +48,7 @@ QT_BEGIN_NAMESPACE
QSGAbstractRendererPrivate::QSGAbstractRendererPrivate()
: m_root_node(nullptr)
, m_clear_color(Qt::transparent)
, m_invertFrontFace(false)
{
m_projection_matrix.resize(1);
m_projection_matrix_native_ndc.resize(1);
@ -302,6 +303,24 @@ QMatrix4x4 QSGAbstractRenderer::projectionMatrixWithNativeNDC(int index) const
return d->m_projection_matrix_native_ndc[index];
}
/*!
\internal
*/
void QSGAbstractRenderer::setInvertFrontFace(bool invert)
{
Q_D(QSGAbstractRenderer);
d->m_invertFrontFace = invert;
}
/*!
\internal
*/
bool QSGAbstractRenderer::invertFrontFace() const
{
Q_D(const QSGAbstractRenderer);
return d->m_invertFrontFace;
}
/*!
Sets the \a color to clear the framebuffer.

View File

@ -60,6 +60,8 @@ public:
QMatrix4x4 projectionMatrixWithNativeNDC(int index) const;
int projectionMatrixCount() const;
int projectionMatrixWithNativeNDCCount() const;
void setInvertFrontFace(bool invert);
bool invertFrontFace() const;
void setClearColor(const QColor &color);
QColor clearColor() const;

View File

@ -42,7 +42,7 @@ public:
QVarLengthArray<QMatrix4x4, 1> m_projection_matrix;
QVarLengthArray<QMatrix4x4, 1> m_projection_matrix_native_ndc;
uint m_mirrored : 1;
uint m_invertFrontFace : 1;
};
QT_END_NAMESPACE

View File

@ -2756,6 +2756,7 @@ bool Renderer::ensurePipelineState(Element *e, const ShaderManager::Shader *sms,
ps->setFlags(flags);
ps->setTopology(qsg_topology(m_gstate.drawMode, m_rhi));
ps->setCullMode(m_gstate.cullMode);
ps->setFrontFace(invertFrontFace() ? QRhiGraphicsPipeline::CW : QRhiGraphicsPipeline::CCW);
ps->setPolygonMode(m_gstate.polygonMode);
ps->setMultiViewCount(m_gstate.multiViewCount);

View File

@ -410,22 +410,41 @@ void QSGRhiLayer::grab()
m_renderer->setDevicePixelRatio(m_dpr);
m_renderer->setDeviceRect(m_pixelSize);
m_renderer->setViewportRect(m_pixelSize);
QRectF mirrored; // in logical coordinates (no dpr) since this gets passed to setProjectionMatrixToRect()
if (m_rhi->isYUpInFramebuffer()) {
// In the unlikely event of back/front face culling used by a custom
// material or effect in the layer, the default front face setting may be
// wrong. Rather, it needs to invert based on what the vertex shader does,
// and so on the rect (and so matrix) generated here.
bool frontFaceSwap = false;
if (m_rhi->isYUpInFramebuffer()) { // basically OpenGL
mirrored = QRectF(m_mirrorHorizontal ? m_logicalRect.right() : m_logicalRect.left(),
m_mirrorVertical ? m_logicalRect.bottom() : m_logicalRect.top(),
m_mirrorHorizontal ? -m_logicalRect.width() : m_logicalRect.width(),
m_mirrorVertical ? -m_logicalRect.height() : m_logicalRect.height());
} else {
if (m_mirrorHorizontal)
frontFaceSwap = !frontFaceSwap;
if (m_mirrorVertical)
frontFaceSwap = !frontFaceSwap;
} else { // APIs other than OpenGL
mirrored = QRectF(m_mirrorHorizontal ? m_logicalRect.right() : m_logicalRect.left(),
m_mirrorVertical ? m_logicalRect.top() : m_logicalRect.bottom(),
m_mirrorHorizontal ? -m_logicalRect.width() : m_logicalRect.width(),
m_mirrorVertical ? m_logicalRect.height() : -m_logicalRect.height());
if (m_mirrorHorizontal)
frontFaceSwap = !frontFaceSwap;
if (!m_mirrorVertical)
frontFaceSwap = !frontFaceSwap;
}
QSGAbstractRenderer::MatrixTransformFlags matrixFlags;
if (!m_rhi->isYUpInNDC())
matrixFlags |= QSGAbstractRenderer::MatrixTransformFlipY;
m_renderer->setProjectionMatrixToRect(mirrored, matrixFlags);
m_renderer->setInvertFrontFace(frontFaceSwap);
m_renderer->setClearColor(Qt::transparent);
m_renderer->setRenderTarget({ m_rt, m_rtRp, m_context->currentFrameCommandBuffer() });

View File

@ -0,0 +1,100 @@
import QtQuick 2.0
Rectangle {
id: topLevel
width: 320
height: 480
// Make it a layer. Important because the winding order inverts based on the transforms the vertex shader does.
// The output should match culling_1 regardless.
layer.enabled: true
ShaderEffectSource {
id: front
visible: false
smooth: true
sourceItem: Rectangle {
width: 256
height: 64
color: "cornflowerblue"
radius: 8
Text {
anchors.centerIn: parent
text: "Front"
font.pixelSize: 48
color: "white"
}
}
}
ShaderEffectSource {
id: back
visible: false
smooth: true
sourceItem: Rectangle {
width: 256
height: 64
color: "firebrick"
radius: 8
Text {
anchors.centerIn: parent
text: "Back"
font.pixelSize: 48
color: "white"
}
}
}
Column {
anchors.fill: parent
Repeater {
model: ListModel {
ListElement {
foo: "No culling"
bar: ShaderEffect.NoCulling
turned: false
}
ListElement {
foo: "Back-face culling"
bar: ShaderEffect.BackFaceCulling
turned: false
}
ListElement {
foo: "Front-face culling"
bar: ShaderEffect.FrontFaceCulling
turned: false
}
}
Item{
id: item_0000
width: 320
height: 120
ShaderEffect {
anchors.right: parent.right
anchors.verticalCenter: parent.verticalCenter
anchors.margins: 10
width: 200
height: 100
cullMode: model.bar
property variant frontSource: front
property variant backSource: back
fragmentShader: "qrc:shaders/culling.frag.qsb"
transform: Rotation {
origin.x: 200
origin.y: 180 - 120 * index
axis { x: 0; y: 1; z: 0 }
angle: (turned == true) ? 180 : 0
}
}
Text {
font.pointSize: 10
anchors.verticalCenter: parent.verticalCenter
anchors.left: parent.left
anchors.margins: 5
text: foo
}
}
}
}
}

View File

@ -0,0 +1,106 @@
import QtQuick 2.0
Rectangle {
id: topLevel
width: 320
height: 480
// Make it a layer. Important because the winding order inverts based on the transforms the vertex shader does.
// The output should match culling_1 regardless.
layer.enabled: true
// layer.textureMirroring by default is MirrorVertically. It has no effect
// on the visual result in this example, but it has a huge effect on the
// transformations internally. Change it to something to see the output is
// still the same.
layer.textureMirroring: ShaderEffectSource.MirrorHorizontally | ShaderEffectSource.MirrorVertically
ShaderEffectSource {
id: front
visible: false
smooth: true
sourceItem: Rectangle {
width: 256
height: 64
color: "cornflowerblue"
radius: 8
Text {
anchors.centerIn: parent
text: "Front"
font.pixelSize: 48
color: "white"
}
}
}
ShaderEffectSource {
id: back
visible: false
smooth: true
sourceItem: Rectangle {
width: 256
height: 64
color: "firebrick"
radius: 8
Text {
anchors.centerIn: parent
text: "Back"
font.pixelSize: 48
color: "white"
}
}
}
Column {
anchors.fill: parent
Repeater {
model: ListModel {
ListElement {
foo: "No culling"
bar: ShaderEffect.NoCulling
turned: false
}
ListElement {
foo: "Back-face culling"
bar: ShaderEffect.BackFaceCulling
turned: false
}
ListElement {
foo: "Front-face culling"
bar: ShaderEffect.FrontFaceCulling
turned: false
}
}
Item{
id: item_0000
width: 320
height: 120
ShaderEffect {
anchors.right: parent.right
anchors.verticalCenter: parent.verticalCenter
anchors.margins: 10
width: 200
height: 100
cullMode: model.bar
property variant frontSource: front
property variant backSource: back
fragmentShader: "qrc:shaders/culling.frag.qsb"
transform: Rotation {
origin.x: 200
origin.y: 180 - 120 * index
axis { x: 0; y: 1; z: 0 }
angle: (turned == true) ? 180 : 0
}
}
Text {
font.pointSize: 10
anchors.verticalCenter: parent.verticalCenter
anchors.left: parent.left
anchors.margins: 5
text: foo
}
}
}
}
}

View File

@ -0,0 +1,106 @@
import QtQuick 2.0
Rectangle {
id: topLevel
width: 320
height: 480
// Make it a layer. Important because the winding order inverts based on the transforms the vertex shader does.
// The output should match culling_1 regardless.
layer.enabled: true
// layer.textureMirroring by default is MirrorVertically. It has no effect
// on the visual result in this example, but it has a huge effect on the
// transformations internally. Change it to something to see the output is
// still the same.
layer.textureMirroring: ShaderEffectSource.MirrorHorizontally
ShaderEffectSource {
id: front
visible: false
smooth: true
sourceItem: Rectangle {
width: 256
height: 64
color: "cornflowerblue"
radius: 8
Text {
anchors.centerIn: parent
text: "Front"
font.pixelSize: 48
color: "white"
}
}
}
ShaderEffectSource {
id: back
visible: false
smooth: true
sourceItem: Rectangle {
width: 256
height: 64
color: "firebrick"
radius: 8
Text {
anchors.centerIn: parent
text: "Back"
font.pixelSize: 48
color: "white"
}
}
}
Column {
anchors.fill: parent
Repeater {
model: ListModel {
ListElement {
foo: "No culling"
bar: ShaderEffect.NoCulling
turned: false
}
ListElement {
foo: "Back-face culling"
bar: ShaderEffect.BackFaceCulling
turned: false
}
ListElement {
foo: "Front-face culling"
bar: ShaderEffect.FrontFaceCulling
turned: false
}
}
Item{
id: item_0000
width: 320
height: 120
ShaderEffect {
anchors.right: parent.right
anchors.verticalCenter: parent.verticalCenter
anchors.margins: 10
width: 200
height: 100
cullMode: model.bar
property variant frontSource: front
property variant backSource: back
fragmentShader: "qrc:shaders/culling.frag.qsb"
transform: Rotation {
origin.x: 200
origin.y: 180 - 120 * index
axis { x: 0; y: 1; z: 0 }
angle: (turned == true) ? 180 : 0
}
}
Text {
font.pointSize: 10
anchors.verticalCenter: parent.verticalCenter
anchors.left: parent.left
anchors.margins: 5
text: foo
}
}
}
}
}

View File

@ -0,0 +1,106 @@
import QtQuick 2.0
Rectangle {
id: topLevel
width: 320
height: 480
// Make it a layer. Important because the winding order inverts based on the transforms the vertex shader does.
// The output should match culling_1 regardless.
layer.enabled: true
// layer.textureMirroring by default is MirrorVertically. It has no effect
// on the visual result in this example, but it has a huge effect on the
// transformations internally. Change it to something to see the output is
// still the same.
layer.textureMirroring: ShaderEffectSource.NoMirroring
ShaderEffectSource {
id: front
visible: false
smooth: true
sourceItem: Rectangle {
width: 256
height: 64
color: "cornflowerblue"
radius: 8
Text {
anchors.centerIn: parent
text: "Front"
font.pixelSize: 48
color: "white"
}
}
}
ShaderEffectSource {
id: back
visible: false
smooth: true
sourceItem: Rectangle {
width: 256
height: 64
color: "firebrick"
radius: 8
Text {
anchors.centerIn: parent
text: "Back"
font.pixelSize: 48
color: "white"
}
}
}
Column {
anchors.fill: parent
Repeater {
model: ListModel {
ListElement {
foo: "No culling"
bar: ShaderEffect.NoCulling
turned: false
}
ListElement {
foo: "Back-face culling"
bar: ShaderEffect.BackFaceCulling
turned: false
}
ListElement {
foo: "Front-face culling"
bar: ShaderEffect.FrontFaceCulling
turned: false
}
}
Item{
id: item_0000
width: 320
height: 120
ShaderEffect {
anchors.right: parent.right
anchors.verticalCenter: parent.verticalCenter
anchors.margins: 10
width: 200
height: 100
cullMode: model.bar
property variant frontSource: front
property variant backSource: back
fragmentShader: "qrc:shaders/culling.frag.qsb"
transform: Rotation {
origin.x: 200
origin.y: 180 - 120 * index
axis { x: 0; y: 1; z: 0 }
angle: (turned == true) ? 180 : 0
}
}
Text {
font.pointSize: 10
anchors.verticalCenter: parent.verticalCenter
anchors.left: parent.left
anchors.margins: 5
text: foo
}
}
}
}
}

View File

@ -0,0 +1,99 @@
import QtQuick 2.0
Rectangle {
id: topLevel
width: 320
height: 480
// Make it a layer. Important because the winding order inverts based on the transforms the vertex shader does.
// The output should match culling_2 regardless.
layer.enabled: true
ShaderEffectSource {
id: front
visible: false
smooth: true
sourceItem: Rectangle {
width: 256
height: 64
color: "cornflowerblue"
radius: 8
Text {
anchors.centerIn: parent
text: "Front"
font.pixelSize: 48
color: "white"
}
}
}
ShaderEffectSource {
id: back
visible: false
smooth: true
sourceItem: Rectangle {
width: 256
height: 64
color: "firebrick"
radius: 8
Text {
anchors.centerIn: parent
text: "Back"
font.pixelSize: 48
color: "white"
}
}
}
Column {
anchors.fill: parent
Repeater {
model: ListModel {
ListElement {
foo: "No culling"
bar: ShaderEffect.NoCulling
turned: true
}
ListElement {
foo: "Back-face culling"
bar: ShaderEffect.BackFaceCulling
turned: true
}
ListElement {
foo: "Front-face culling"
bar: ShaderEffect.FrontFaceCulling
turned: true
}
}
Item{
id: item_0000
width: 320
height: 120
ShaderEffect{
anchors.right: parent.right
anchors.verticalCenter: parent.verticalCenter
anchors.margins: 10
width: 200
height: 100
cullMode: model.bar
property variant frontSource: front
property variant backSource: back
fragmentShader: "qrc:shaders/culling.frag.qsb"
transform: Rotation {
origin.x: 100
origin.y: 180 - 120 * index
axis { x: 0; y: 1; z: 0 }
angle: (turned == true) ? 180 : 0
}
}
Text {
font.pointSize: 10
anchors.verticalCenter: parent.verticalCenter
anchors.left: parent.left
anchors.margins: 5
text: foo
}
}
}
}
}