Stencil shadows problem needs solving

Blitz3D Forums/Blitz3D Programming/Stencil shadows problem needs solving

The forum is locked. These are the original blitzbasic.com forums, kept so they can still be read. No new topics, replies or edits can be made here. To join in, head over to the Blitz3D+ forums.

Braincell
 #1
This is as far as i've got if anyone is interested... (just some more messy code but making progress).

Now i can make the volumes work great only the shadows add to eachother making it look like the objects above are
always transparent or are coming from 2 different lights which is ok to have control over in some cases but not in
the current case, its not what i'm looking for.

I keep spamming this thread cos i'm waiting for someone to give suggestions. Still, anyone?

;WARNING: Messy code ahead. May cause severe nausia. Consult your doctor b4 viewing.

;D3DCULL
Const D3DCULL_NONE				= 1
Const D3DCULL_CW					= 2
Const D3DCULL_CCW					= 3
		
;D3DSHADE_MODE
Const D3DSHADE_FLAT			= 1
Const D3DSHADE_GOURAUD	= 2
Const D3DSHADE_PHONG		= 3

;D3DFILL_MODE
Const D3DFILL_POINT			= 1
Const D3DFILL_WIREFRAME	= 2
Const D3DFILL_SOLID			= 3

;D3DSTENCILOP
Const D3DSTENCILOP_KEEP           = 1
Const D3DSTENCILOP_ZERO           = 2
Const D3DSTENCILOP_REPLACE        = 3
Const D3DSTENCILOP_INCRSAT        = 4
Const D3DSTENCILOP_DECRSAT        = 5
Const D3DSTENCILOP_INVERT         = 6
Const D3DSTENCILOP_INCR           = 7
Const D3DSTENCILOP_DECR           = 8
Const D3DSTENCILOP_FORCE_DWORD    = $7fffffff	;force 32-bit size enum

;D3DCMPFUNC
Const D3DCMP_NEVER               = 1
Const D3DCMP_LESS                = 2
Const D3DCMP_EQUAL               = 3
Const D3DCMP_LESSEQUAL           = 4
Const D3DCMP_GREATER             = 5
Const D3DCMP_NOTEQUAL            = 6
Const D3DCMP_GREATEREQUAL        = 7
Const D3DCMP_ALWAYS              = 8
Const D3DCMP_FORCE_DWORD         = $7fffffff	;force 32-bit size enum

;D3DBLEND
Const D3DBLEND_ZERO              = 1
Const D3DBLEND_ONE               = 2
Const D3DBLEND_SRCCOLOR          = 3
Const D3DBLEND_INVSRCCOLOR       = 4
Const D3DBLEND_SRCALPHA          = 5
Const D3DBLEND_INVSRCALPHA       = 6
Const D3DBLEND_DESTALPHA         = 7
Const D3DBLEND_INVDESTALPHA      = 8
Const D3DBLEND_DESTCOLOR         = 9
Const D3DBLEND_INVDESTCOLOR      = 10
Const D3DBLEND_SRCALPHASAT       = 11
Const D3DBLEND_BOTHSRCALPHA      = 12
Const D3DBLEND_BOTHINVSRCALPHA   = 13
Const D3DBLEND_FORCE_DWORD       = $7fffffff	;force 32-bit size enum



Const D3DRS_ANTIALIAS          = 2    ; D3DANTIALIASMODE */
Const D3DRS_TEXTUREPERSPECTIVE = 4    ; True For perspective correction */
Const D3DRS_ZENABLE            = 7    ; D3DZBUFFERTYPE (Or True/False For legacy) */
Const D3DRS_FILLMODE           = 8    ; D3DFILL_MODE        */
Const D3DRS_SHADEMODE          = 9    ; D3DFILL_MODE */
Const D3DRS_LINEPATTERN        = 10   ; D3DLINEPATTERN */
Const D3DRS_ZWRITEENABLE       = 14   ; True To enable z writes */
Const D3DRS_ALPHATESTENABLE    = 15   ; True To enable alpha tests */
Const D3DRS_LASTPIXEL          = 16   ; True For Last-pixel on lines */
Const D3DRS_SRCBLEND           = 19   ; D3DBLEND */
Const D3DRS_DESTBLEND          = 20   ; D3DBLEND */
Const D3DRS_CULLMODE           = 22   ; D3DCULL */
Const D3DRS_ZFUNC              = 23   ; D3DCMPFUNC */
Const D3DRS_ALPHAREF           = 24   ; D3DFIXED (long) */
Const D3DRS_ALPHAFUNC          = 25   ; D3DCMPFUNC */
Const D3DRS_DITHERENABLE       = 26   ; True To enable dithering */
Const D3DRS_ALPHABLENDENABLE   = 27   ; True To enable alpha blending */
Const D3DRS_FOGENABLE          = 28   ; True To enable fog blending */
Const D3DRS_SPECULARENABLE     = 29   ; True To enable specular */
Const D3DRS_ZVISIBLE           = 30   ; True To enable z checking */
Const D3DRS_STIPPLEDALPHA      = 33   ; True To enable stippled alpha (RGB device only) */
Const D3DRS_FOGCOLOR           = 34   ; D3DCOLOR */  INT (((a) << 24) Or ((r) << 16) Or ((g) << 8) Or (b)))
Const D3DRS_FOGTABLEMODE       = 35   ; D3DFOGMODE */
Const D3DRS_FOGSTART           = 36   ; Float Fog start (For both vertex And pixel fog) 
Const D3DRS_FOGEND             = 37   ; Float Fog End      */
Const D3DRS_FOGDENSITY         = 38   ; Fog density  */
Const D3DRS_EDGEANTIALIAS      = 40   ; True To enable edge antialiasing */
Const D3DRS_COLORKEYENABLE     = 41   ; True To enable source colorkeyed textures */
Const D3DRS_ZBIAS              = 47   ; LONG Z bias */
Const D3DRS_RANGEFOGENABLE     = 48   ; Enables range-based fog */

; *** STENCIL OPS ***
Const D3DRS_STENCILENABLE      = 52   ; BOOL enable/disable stenciling
Const D3DRS_STENCILFAIL        = 53   ; D3DSTENCILOP To do If stencil test fails
Const D3DRS_STENCILZFAIL       = 54   ; D3DSTENCILOP To do If stencil test passes And Z test fails
Const D3DRS_STENCILPASS        = 55   ; D3DSTENCILOP To do If both stencil And Z tests pass */
Const D3DRS_STENCILFUNC        = 56   ; D3DCMPFUNC fn.  Stencil Test passes If ((ref & mask) stencilfn (stencil & mask)) is True */
Const D3DRS_STENCILREF         = 57   ; INT Reference value used in stencil test */
Const D3DRS_STENCILMASK        = 58   ; Mask value used in stencil test  e.g (0xffffffff)
Const D3DRS_STENCILWRITEMASK   = 59   ; Write mask applied To values written To stencil buffer e.g (0xffffffff)
Const D3DRS_TEXTUREFACTOR      = 60   ; D3DCOLOR used For multi-texture blend */

Const D3DCLEAR_TARGET					= $00000001	;Clear target surface
Const D3DCLEAR_ZBUFFER				= $00000002	;Clear target z buffer
Const D3DCLEAR_STENCIL				= $00000004	;Clear Stencil


;ERRORS
Const D3DERR_ZBUFFER_NOTPRESENT					= 2070
Const D3DERR_STENCILBUFFER_NOTPRESENT		= 2071
Const D3DERR_VIEWPORTHASNODEVICE				= 774
Const DDERR_INVALIDOBJECT								= 130
Const DDERR_INVALIDPARAMS								= $80070057

Type BBRect
	Field x1%
	Field lX1%

	Field y1%
	Field lY1%

	Field x2%
	Field lX2%
	
	Field y2%
  Field lY2%
End Type

Graphics3D 1027,768,32,2

Direct3D7=SystemProperty$("Direct3D7")
Direct3DDevice7=SystemProperty$("Direct3DDevice7")
DirectDraw7=SystemProperty$("DirectDraw7")
DirectInput7=SystemProperty$("DirectInput7")
AppHWND=SystemProperty$("AppHWND")
AppHINSTANCE=SystemProperty$("AppHINSTANCE")




cam=CreateCamera()
piv=CreatePivot()
PositionEntity cam,-5,3,-5
;RotateEntity cam,30,0,0
PointEntity cam,piv
CameraRange cam,.1,50

light=CreateLight()
RotateEntity light,60,30,0

tex=CreateTexture(512,512)
ScaleTexture tex,.2,.5
SetBuffer TextureBuffer(tex)
Color 100,100,100
Rect 0,0,512,512
Color 200,200,200
Rect 8,8,496,496
Color 255,255,255
SetBuffer BackBuffer()

cyl1=CreateCylinder()
EntityAlpha cyl1,.3
;EntityFX cyl1,1
PositionEntity cyl1,2,0,0
ScaleEntity cyl1,3,3,3

cyl2=CreateCylinder()
EntityAlpha cyl2,.3
PositionEntity cyl2,2,0,5
ScaleEntity cyl2,3,3,3

cube=CreateCube()
PositionEntity cube,0,0,0
ScaleEntity cube,5,.2,10
EntityTexture cube,tex


;cube2=CreateCube()
;EntityColor cube2,200,200,200
;PositionEntity cube2,0,0,0
;EntityTexture cube2,tex

;cyl=CreateCylinder()
;PositionEntity cyl,-3,1,0
;RotateMesh cyl,90,0,0
;ScaleEntity cyl,.2,.2,1



spa#=.3
shadowplane=CreateCube() ; squished Z cube
ScaleMesh shadowplane,10,10,.01
EntityParent shadowplane,cam,0
MoveEntity shadowplane,0,0,.2
EntityColor shadowplane,0,0,0
EntityAlpha shadowplane,spa#


sv=1

; AREA TO CLEAR
r.BBRect=New BBRect
r\lX1=0
r\lY1=0
;r\lX2=GraphicsWidth()/2 ; half the screen
r\lX2=GraphicsWidth() ; half the screen
r\lY2=GraphicsHeight()


CameraClsMode cam,False,False

While Not KeyHit(1)

.start
	PositionEntity cyl1,(MouseX()-(GraphicsWidth()*.5)) * .01,-(MouseY()-(GraphicsWidth()*.5)) * .02,0
	PositionEntity cyl2,(MouseX()-(GraphicsWidth()*.5)) * .01,-(MouseY()-(GraphicsWidth()*.5)) * .02,2
	
	
	t=MilliSecs()

	;----------------------------------render start	
	; Render Scene
	HideEntity shadowplane
	HideEntity cyl1
	ShowEntity cube
;	ShowEntity cube2
;	ShowEntity cyl

	SetRenderState(Direct3DDevice7, D3DRS_ZWRITEENABLE, True)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILENABLE, False)
	SetRenderState(Direct3DDevice7, D3DRS_ALPHABLENDENABLE, False)


	;DeviceClear(Direct3DDevice7,   D3DCLEAR_ZBUFFER Or D3DCLEAR_STENCIL,$00008080,1.0,$00000000)
	DeviceClear2(Direct3DDevice7, r, D3DCLEAR_TARGET Or D3DCLEAR_ZBUFFER Or D3DCLEAR_STENCIL,$00008080,1.0,0)
	RenderWorld


	ShowEntity cyl1
	HideEntity cube
;	HideEntity cube2
;	HideEntity cyl

	SetRenderState(Direct3DDevice7, D3DRS_ZWRITEENABLE,False)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILENABLE,True)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILFUNC, D3DCMP_ALWAYS)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILREF, 1 )
	SetRenderState(Direct3DDevice7, D3DRS_STENCILMASK,$ffffffff)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILWRITEMASK,$ffffffff)

	; If ztest passes, Write 1 into stencil buffer
	SetRenderState(Direct3DDevice7, D3DRS_STENCILZFAIL, D3DSTENCILOP_KEEP)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILFAIL,  D3DSTENCILOP_KEEP)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILPASS,  D3DSTENCILOP_REPLACE)

	; Make sure that no pixels get drawn to the frame buffer
	SetRenderState(Direct3DDevice7, D3DRS_ALPHABLENDENABLE, True )
	SetRenderState(Direct3DDevice7, D3DRS_SRCBLEND,  D3DBLEND_ZERO )
	SetRenderState(Direct3DDevice7, D3DRS_DESTBLEND, D3DBLEND_ONE)

	; Draw front-side of shadow volume in stencil/z only
	RenderWorld


	; Draw back side shadow polys in Z only
	;FlipMesh cyl1
	SetRenderState(Direct3DDevice7, D3DRS_STENCILREF, 0)
	SetRenderState(Direct3DDevice7, D3DRS_CULLMODE,   D3DCULL_CW ) ; use flipmesh instead
	RenderWorld
	SetRenderState(Direct3DDevice7, D3DRS_CULLMODE, D3DCULL_CCW )  ; use flipmesh instead
	;FlipMesh cyl1
	HideEntity cyl1
	RenderWorld

  ; Restore render states
	SetRenderState(Direct3DDevice7, D3DRS_ZWRITEENABLE,     True )
	SetRenderState(Direct3DDevice7, D3DRS_STENCILENABLE,    False )
	SetRenderState(Direct3DDevice7, D3DRS_ALPHABLENDENABLE, False )

; *** DRAW SHADOW ***
	SetRenderState(Direct3DDevice7, D3DRS_ZENABLE,       False )
	SetRenderState(Direct3DDevice7, D3DRS_STENCILENABLE, True )

    ; Turn on alphablending
	SetRenderState(Direct3DDevice7, D3DRS_ALPHABLENDENABLE, True )
	SetRenderState(Direct3DDevice7, D3DRS_SRCBLEND, D3DBLEND_SRCALPHA )
	SetRenderState(Direct3DDevice7, D3DRS_DESTBLEND, D3DBLEND_INVSRCALPHA )

    ; Only Write where the stencil value == 1
	SetRenderState(Direct3DDevice7, D3DRS_STENCILREF,  1 )
	SetRenderState(Direct3DDevice7, D3DRS_STENCILFUNC, D3DCMP_EQUAL )
	SetRenderState(Direct3DDevice7, D3DRS_STENCILPASS, D3DSTENCILOP_REPLACE )

	
	ShowEntity shadowplane
	RenderWorld

	; Restore Render State
	SetRenderState(Direct3DDevice7, D3DRS_ZENABLE,          True )
	SetRenderState(Direct3DDevice7, D3DRS_STENCILENABLE,    False )
	SetRenderState(Direct3DDevice7, D3DRS_ALPHABLENDENABLE, True )

	ShowEntity cyl1
	HideEntity shadowplane

	RenderWorld;----------------------------------render end
	HideEntity cyl1
	;----------------------------------render start	
	; Render Scene
;	HideEntity shadowplane
;	HideEntity cyl2
;	ShowEntity cube
;	ShowEntity cube2
;	ShowEntity cyl
;
;	SetRenderState(Direct3DDevice7, D3DRS_ZWRITEENABLE, True)
;	SetRenderState(Direct3DDevice7, D3DRS_STENCILENABLE, False)
;	SetRenderState(Direct3DDevice7, D3DRS_ALPHABLENDENABLE, False)
;
;
	;DeviceClear(Direct3DDevice7,   D3DCLEAR_ZBUFFER Or D3DCLEAR_STENCIL,$00008080,1.0,$00000000)
;	DeviceClear2(Direct3DDevice7, r, D3DCLEAR_TARGET Or D3DCLEAR_ZBUFFER Or D3DCLEAR_STENCIL,$00008080,1.0,0)
;	RenderWorld
;
DeviceClear2(Direct3DDevice7, r, D3DCLEAR_STENCIL,$00008080,1.0,0)


	ShowEntity cyl2
	HideEntity cube
;	HideEntity cube2
;	HideEntity cyl

	SetRenderState(Direct3DDevice7, D3DRS_ZWRITEENABLE,False)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILENABLE,True)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILFUNC, D3DCMP_ALWAYS)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILREF, 1 )
	SetRenderState(Direct3DDevice7, D3DRS_STENCILMASK,$ffffffff)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILWRITEMASK,$ffffffff)

	; If ztest passes, Write 1 into stencil buffer
	SetRenderState(Direct3DDevice7, D3DRS_STENCILZFAIL, D3DSTENCILOP_KEEP)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILFAIL,  D3DSTENCILOP_KEEP)
	SetRenderState(Direct3DDevice7, D3DRS_STENCILPASS,  D3DSTENCILOP_REPLACE)

	; Make sure that no pixels get drawn to the frame buffer
	SetRenderState(Direct3DDevice7, D3DRS_ALPHABLENDENABLE, True )
	SetRenderState(Direct3DDevice7, D3DRS_SRCBLEND,  D3DBLEND_ZERO )
	SetRenderState(Direct3DDevice7, D3DRS_DESTBLEND, D3DBLEND_ONE)

	; Draw front-side of shadow volume in stencil/z only
	RenderWorld


	; Draw back side shadow polys in Z only
	;FlipMesh cyl2
	SetRenderState(Direct3DDevice7, D3DRS_STENCILREF, 0)
	SetRenderState(Direct3DDevice7, D3DRS_CULLMODE,   D3DCULL_CW ) ; use flipmesh instead
	RenderWorld
	SetRenderState(Direct3DDevice7, D3DRS_CULLMODE, D3DCULL_CCW )  ; use flipmesh instead
	;FlipMesh cyl2
	HideEntity cyl2
	RenderWorld

  ; Restore render states
	SetRenderState(Direct3DDevice7, D3DRS_ZWRITEENABLE,     True )
	SetRenderState(Direct3DDevice7, D3DRS_STENCILENABLE,    False )
	SetRenderState(Direct3DDevice7, D3DRS_ALPHABLENDENABLE, False )

; *** DRAW SHADOW ***
	SetRenderState(Direct3DDevice7, D3DRS_ZENABLE,       False )
	SetRenderState(Direct3DDevice7, D3DRS_STENCILENABLE, True )

    ; Turn on alphablending
	SetRenderState(Direct3DDevice7, D3DRS_ALPHABLENDENABLE, True )
	SetRenderState(Direct3DDevice7, D3DRS_SRCBLEND, D3DBLEND_SRCALPHA )
	SetRenderState(Direct3DDevice7, D3DRS_DESTBLEND, D3DBLEND_INVSRCALPHA )

    ; Only Write where the stencil value == 1
	SetRenderState(Direct3DDevice7, D3DRS_STENCILREF,  1 )
	SetRenderState(Direct3DDevice7, D3DRS_STENCILFUNC, D3DCMP_EQUAL )
	SetRenderState(Direct3DDevice7, D3DRS_STENCILPASS, D3DSTENCILOP_REPLACE )

	
	ShowEntity shadowplane
	RenderWorld

	; Restore Render State
	SetRenderState(Direct3DDevice7, D3DRS_ZENABLE,          True )
	SetRenderState(Direct3DDevice7, D3DRS_STENCILENABLE,    False )
	SetRenderState(Direct3DDevice7, D3DRS_ALPHABLENDENABLE, True )

	ShowEntity cyl2
	HideEntity shadowplane

	RenderWorld;----------------------------------render end
	If tra=True HideEntity cyl2

	SetRenderState(Direct3DDevice7, D3DRS_ALPHABLENDENABLE, False )
	Text 0,0,"MS: "+(MilliSecs() - t)
	Text 0,20,"Shadow Alpha: "+spa+ "    LMB/RMB to change value, SPACE to toggle visibility"
	Text 0,30, "Press O to toggle object 'transparency'"

	If MouseDown(1)
		spa=spa-.01
		If spa<0 spa=0
		EntityAlpha shadowplane,spa
	Else If MouseDown(2)
		spa=spa+.01
		If spa>1.0 spa=1.0
		EntityAlpha shadowplane,spa
	End If

	If KeyHit(57)
		sv=Not sv
		If sv
			EntityAlpha cyl1,.3
			EntityAlpha cyl2,.3
		Else
			EntityAlpha cyl1,0.001
			EntityAlpha cyl2,0.001			
		End If
	End If
	
	If KeyHit(24) tra=Not tra 
	
;	TurnEntity cyl,0,1,0
;	TurnEntity cube2,0,0,.5


Flip

Wend
End