Resolution: a new shader and its material, plus one additional material. Shader creation can be avoided in some situations!
Shaders in Unity has a tool called Stencil. This tool maintains pixel buffer that can be used to discard pixels.
CanvasImageMask shader: it is a clone of default UI image shader, except Stencil adjustments and tiny addition for comfort work. When mask rendering is happening, Stencil will set the pixel buffer's value to 1 in the place where it actually drew something. The second shader reacts to this change and discards new pixels in that location. Keep in mind that objects with such material (that hold CanvasImageMask shader) should come first in Canvas because during rendering they will modify pixel buffers, which can be used later.
All objects that should be masked have to use another material that is based on default UI image shader with Stencil adjustments. This material will discard any pixels with a buffer's value equal or higher than 1.
Note that this work can be avoided. The developer can create 2 materials (clones of default image material), and modify Stencil settings in the editor. But this solution has a problem: how to create such a mask, that shouldn't be drawn? The best solution would be alpha channels set to 1 (out of 255). But if Canvas is using CanvasGroup, pixels can be discarded earlier, and the mask won't work.
CanvasImageMask shader hides Stencil settings (convenience) and adds a flag, which will hide all pixels from the mask yet still do the job.
CanvasImageMask:
Shader "Custom/CanvasImageMask"
{
Properties
{
[PerRendererData] _MainTex ("Sprite Texture", 2D) = "white" {}
_Color ("Tint", Color) = (1,1,1,1)
[MaterialToggle(DO_NOT_DRAW)]
_DoNotDraw ("Do not draw", Float) = 0
}
SubShader
{
Tags
{
"Queue"="Transparent"
"IgnoreProjector"="True"
"RenderType"="Transparent"
"PreviewType"="Plane"
"CanUseSpriteAtlas"="True"
}
Stencil
{
Ref 1
Comp Greater
Pass Replace
}
Cull Off
Lighting Off
ZWrite Off
ZTest [unity_GUIZTestMode]
Blend SrcAlpha OneMinusSrcAlpha
ColorMask RGBA
Pass
{
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#pragma shader_feature DO_NOT_DRAW
#include "UnityCG.cginc"
struct appdata_t
{
float4 vertex : POSITION;
float4 color : COLOR;
float2 texcoord : TEXCOORD0;
};
struct v2f
{
float4 vertex : SV_POSITION;
fixed4 color : COLOR;
half2 texcoord : TEXCOORD0;
};
fixed4 _Color;
v2f vert(appdata_t IN)
{
v2f OUT;
OUT.vertex = UnityObjectToClipPos(IN.vertex);
OUT.texcoord = IN.texcoord;
#ifdef UNITY_HALF_TEXEL_OFFSET
OUT.vertex.xy += (_ScreenParams.zw-1.0)*float2(-1,1);
#endif
OUT.color = IN.color * _Color;
return OUT;
}
sampler2D _MainTex;
fixed4 frag(v2f IN) : SV_Target
{
half4 color = tex2D(_MainTex, IN.texcoord) * IN.color;
clip (color.a - 0.01);
#ifdef DO_NOT_DRAW
color.a = 0;
#endif
return color;
}
ENDCG
}
}
}
If you want to compare, here is the default UI image shader:
Shader "UI/Default"
{
Properties
{
[PerRendererData] _MainTex ("Sprite Texture", 2D) = "white" {}
_Color ("Tint", Color) = (1,1,1,1)
_StencilComp ("Stencil Comparison", Float) = 8
_Stencil ("Stencil ID", Float) = 0
_StencilOp ("Stencil Operation", Float) = 0
_StencilWriteMask ("Stencil Write Mask", Float) = 255
_StencilReadMask ("Stencil Read Mask", Float) = 255
_ColorMask ("Color Mask", Float) = 15
}
SubShader
{
Tags
{
"Queue"="Transparent"
"IgnoreProjector"="True"
"RenderType"="Transparent"
"PreviewType"="Plane"
"CanUseSpriteAtlas"="True"
}
Stencil
{
Ref [_Stencil]
Comp [_StencilComp]
Pass [_StencilOp]
ReadMask [_StencilReadMask]
WriteMask [_StencilWriteMask]
}
Cull Off
Lighting Off
ZWrite Off
ZTest [unity_GUIZTestMode]
Blend SrcAlpha OneMinusSrcAlpha
ColorMask [_ColorMask]
Pass
{
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
struct appdata_t
{
float4 vertex : POSITION;
float4 color : COLOR;
float2 texcoord : TEXCOORD0;
};
struct v2f
{
float4 vertex : SV_POSITION;
fixed4 color : COLOR;
half2 texcoord : TEXCOORD0;
};
fixed4 _Color;
v2f vert(appdata_t IN)
{
v2f OUT;
OUT.vertex = UnityObjectToClipPos(IN.vertex);
OUT.texcoord = IN.texcoord;
#ifdef UNITY_HALF_TEXEL_OFFSET
OUT.vertex.xy += (_ScreenParams.zw-1.0)*float2(-1,1);
#endif
OUT.color = IN.color * _Color;
return OUT;
}
sampler2D _MainTex;
fixed4 frag(v2f IN) : SV_Target
{
half4 color = tex2D(_MainTex, IN.texcoord) * IN.color;
clip (color.a - 0.01);
return color;
}
ENDCG
}
}
}
To summarize: create CanvasImageMask, create material based on this shader, and place an image on the canvas. Create another material, based on default shader, set Stencil ID to 1, and Stencil comparison to 5 (aka "Greater"). Assign this material to any other image on the canvas that comes after mask object.
References: