OM Depth-Stencil Samples

The Output Merger (OM) stage is the final stage in the DirectX rendering pipeline. It handles tasks such as depth testing, stencil testing, blending, and writing to the render target and depth-stencil buffers. This section provides practical code samples demonstrating common depth-stencil operations.

Depth Testing

Depth testing is crucial for rendering 3D scenes correctly, ensuring that objects closer to the camera obscure objects farther away. This sample demonstrates basic depth testing setup.

// Create Depth-Stencil View Description
D3D11_DEPTH_STENCIL_VIEW_DESC dsvDesc;
ZeroMemory(&dsvDesc, sizeof(dsvDesc));
dsvDesc.Format = DXGI_FORMAT_D24_UNORM_S8_UINT; // Or your preferred format
dsvDesc.ViewDimension = D3D11_DSV_DIMENSION_TEXTURE2D;
dsvDesc.Texture2D.MipSlice = 0;

// Create the Depth-Stencil Texture
ID3D11Texture2D* pDepthStencilBuffer = nullptr;
// ... texture creation code ...

// Create the Depth-Stencil View
ID3D11DepthStencilView* pDepthStencilView = nullptr;
// pDevice->CreateDepthStencilView(pDepthStencilBuffer, &dsvDesc, &pDepthStencilView);

// Create Depth-Stencil State Description
D3D11_DEPTH_STENCIL_DESC dsStateDesc;
ZeroMemory(&dsStateDesc, sizeof(dsStateDesc));

// Enable depth testing
dsStateDesc.DepthEnable = true;
dsStateDesc.DepthWriteMask = D3D11_DEPTH_WRITE_MASK_ALL;
dsStateDesc.DepthFunc = D3D11_COMPARISON_LESS; // Typical for depth testing

// Stencil testing is optional and disabled by default here
dsStateDesc.StencilEnable = false;
// ... other stencil properties ...

// Create the Depth-Stencil State
ID3D11DepthStencilState* pDepthStencilState = nullptr;
// pDevice->CreateDepthStencilState(&dsStateDesc, &pDepthStencilState);

// Bind the Depth-Stencil View and State to the pipeline
// pContext->OMSetRenderTargets(1, &pRenderTargetView, pDepthStencilView);
// pContext->OMSetDepthStencilState(pDepthStencilState, 1); // Stencil reference value (ignored if StencilEnable is false)
                

Stencil Testing

Stencil testing provides a more advanced way to control rendering based on values stored in the stencil buffer. This is often used for effects like shadow volumes, mirror reflections, or masking.

Basic Stencil Buffer Usage (Masking)

This example shows how to use the stencil buffer to draw only within a specific region.

// Create Depth-Stencil State Description for Stencil Masking
D3D11_DEPTH_STENCIL_DESC dsStateDescMask;
ZeroMemory(&dsStateDescMask, sizeof(dsStateDescMask));

dsStateDescMask.DepthEnable = true; // Depth testing can still be active
dsStateDescMask.DepthWriteMask = D3D11_DEPTH_WRITE_MASK_ALL;
dsStateDescMask.DepthFunc = D3D11_COMPARISON_LESS;

dsStateDescMask.StencilEnable = true;

// Front-face stencil operations
dsStateDescMask.FrontFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;
dsStateDescMask.FrontFace.StencilDepthFailOp = D3D11_STENCIL_OP_KEEP;
dsStateDescMask.FrontFace.StencilPassOp = D3D11_STENCIL_OP_REPLACE; // Write to stencil buffer
dsStateDescMask.FrontFace.StencilFunc = D3D11_COMPARISON_ALWAYS;   // Always pass stencil test

// Back-face stencil operations (usually same as front for simple masking)
dsStateDescMask.BackFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;
dsStateDescMask.BackFace.StencilDepthFailOp = D3D11_STENCIL_OP_KEEP;
dsStateDescMask.BackFace.StencilPassOp = D3D11_STENCIL_OP_REPLACE;
dsStateDescMask.BackFace.StencilFunc = D3D11_COMPARISON_ALWAYS;

// Create the Depth-Stencil State for masking
ID3D11DepthStencilState* pStencilMaskState = nullptr;
// pDevice->CreateDepthStencilState(&dsStateDescMask, &pStencilMaskState);

// --- First Pass: Draw a shape to set stencil values ---
// Set stencil reference value (e.g., 1)
// pContext->OMSetDepthStencilState(pStencilMaskState, 1); 

// Disable depth writing to avoid writing to depth buffer during stencil setup
D3D11_DEPTH_STENCIL_DESC dsStateDescNoDepthWrite;
ZeroMemory(&dsStateDescNoDepthWrite, sizeof(dsStateDescNoDepthWrite));
dsStateDescNoDepthWrite.DepthEnable = true; // Keep depth testing if needed
dsStateDescNoDepthWrite.DepthWriteMask = D3D11_DEPTH_WRITE_MASK_ZERO; // No depth writes
dsStateDescNoDepthWrite.DepthFunc = D3D11_COMPARISON_LESS;
dsStateDescNoDepthWrite.StencilEnable = true;
dsStateDescNoDepthWrite.FrontFace.StencilFunc = D3D11_COMPARISON_ALWAYS;
dsStateDescNoDepthWrite.FrontFace.StencilPassOp = D3D11_STENCIL_OP_REPLACE;
dsStateDescNoDepthWrite.FrontFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;
dsStateDescNoDepthWrite.FrontFace.StencilDepthFailOp = D3D11_STENCIL_OP_KEEP;
// ... copy back-face settings ...
ID3D11DepthStencilState* pNoDepthWriteState = nullptr;
// pDevice->CreateDepthStencilState(&dsStateDescNoDepthWrite, &pNoDepthWriteState);
// pContext->OMSetDepthStencilState(pNoDepthWriteState, 1); 
// ... draw geometry that defines the masked region ...

// --- Second Pass: Draw main scene, masked by stencil ---
// Configure stencil state to only pass if stencil buffer matches reference
D3D11_DEPTH_STENCIL_DESC dsStateDescMaskedDraw;
ZeroMemory(&dsStateDescMaskedDraw, sizeof(dsStateDescMaskedDraw));
dsStateDescMaskedDraw.DepthEnable = true;
dsStateDescMaskedDraw.DepthWriteMask = D3D11_DEPTH_WRITE_MASK_ALL;
dsStateDescMaskedDraw.DepthFunc = D3D11_COMPARISON_LESS;

dsStateDescMaskedDraw.StencilEnable = true;
dsStateDescMaskedDraw.FrontFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;
dsStateDescMaskedDraw.FrontFace.StencilDepthFailOp = D3D11_STENCIL_OP_KEEP;
dsStateDescMaskedDraw.FrontFace.StencilPassOp = D3D11_STENCIL_OP_KEEP; // Do nothing to stencil
dsStateDescMaskedDraw.FrontFace.StencilFunc = D3D11_COMPARISON_EQUAL;  // Only draw if stencil value equals reference

// ... copy back-face settings ...
ID3D11DepthStencilState* pMaskedDrawState = nullptr;
// pDevice->CreateDepthStencilState(&dsStateDescMaskedDraw, &pMaskedDrawState);
// pContext->OMSetDepthStencilState(pMaskedDrawState, 1); // Reference value is 1
// ... draw the main scene geometry ...
                

Depth Bias

Depth bias is used to prevent rendering artifacts, such as Z-fighting, which occurs when two surfaces have very similar depths, leading to flickering. Depth bias adds or subtracts a small value from the depth of a pixel during depth testing.

// Create Rasterizer State Description
D3D11_RASTERIZER_DESC rsDesc;
ZeroMemory(&rsDesc, sizeof(rsDesc));

// Enable depth bias
rsDesc.DepthBias = 10000; // A small integer value. The actual bias depends on depth precision.
rsDesc.DepthBiasClamp = 0.0f; // Clamps bias to this value.
rsDesc.SlopeScaledDepthBias = 1.0f; // Scales bias based on the slope of the triangle.

// ... other rasterizer state properties ...

// Create the Rasterizer State
ID3D11RasterizerState* pRasterizerState = nullptr;
// pDevice->CreateRasterizerState(&rsDesc, &pRasterizerState);

// Bind the Rasterizer State to the pipeline
// pContext->RSSetState(pRasterizerState);
                

Blending

While not strictly part of depth-stencil operations, blending is handled by the Output Merger stage and is often used in conjunction with depth testing (e.g., for transparent objects). Blending combines the newly computed pixel color with the existing color in the render target.

// Create Blend State Description
D3D11_BLEND_DESC blendDesc;
ZeroMemory(&blendDesc, sizeof(blendDesc));

// Enable blending for the render target
blendDesc.AlphaToCoverageEnable = false;
blendDesc.RenderTarget[0].BlendEnable = true;
blendDesc.RenderTarget[0].SrcBlend = D3D11_BLEND_SRC_ALPHA; // Source alpha
blendDesc.RenderTarget[0].DestBlend = D3D11_BLEND_INV_SRC_ALPHA; // Inverse of source alpha
blendDesc.RenderTarget[0].BlendOp = D3D11_BLEND_OP_ADD; // Operation to combine
blendDesc.RenderTarget[0].SrcBlendAlpha = D3D11_BLEND_ONE;
blendDesc.RenderTarget[0].DestBlendAlpha = D3D11_BLEND_ZERO;
blendDesc.RenderTarget[0].BlendOpAlpha = D3D11_BLEND_OP_ADD;

// Create the Blend State
ID3D11BlendState* pBlendState = nullptr;
// pDevice->CreateBlendState(&blendDesc, &pBlendState);

// Set blend factors (optional, used if SrcBlend/DestBlend is D3D11_BLEND_BLEND_FACTOR)
float blendFactor[] = { 0.0f, 0.0f, 0.0f, 0.0f };

// Bind the Blend State and factors
// pContext->OMSetBlendState(pBlendState, blendFactor, 0xFFFFFFFF);