fun rotation math that i just cant get right

here is an example in MiniB3D. use arrow keys to move the right player, WASD for the left player:

[EDIT] i was able to get it working finally the original way i was trying to accomplish it and provided the faceeach2 function below. the mathematical way you were trying to do it is probably more effecient/fast so if you feel like taking a crack at it please do. if not, i think im good! thank you very much for your help :)

Framework SiDesign.MiniB3D

Graphics3D 640,480,0,2
Local camera:TCamera = CreateCamera()
PositionEntity camera,0,25,-25
RotateEntity camera,55,0,0

Local face1:face = face.Create()
Local face2:face = face.Create()

PositionEntity face1.Entity(),-17, 0, 17
RotateEntity face1.Entity(), 0, -90, 0

PositionEntity face2.Entity(),17, 0, -17
RotateEntity face2.Entity(), 0, 90, 0

While Not KeyDown( KEY_ESCAPE )
	
	If KeyDown(KEY_UP) Then MoveEntity(face2.Entity(), 0, 0 ,1)
	If KeyDown(KEY_DOWN) Then MoveEntity(face2.Entity(), 0, 0 ,-1)
	If KeyDown(KEY_LEFT) Then TurnEntity(face2.Entity(), 0, 5, 0)
	If KeyDown(KEY_RIGHT) Then TurnEntity(face2.Entity(), 0, -5, 0)

	If KeyDown(KEY_W) Then MoveEntity(face1.Entity(), 0, 0 ,1)
	If KeyDown(KEY_A) Then TurnEntity(face1.Entity(), 0, 5, 0)
	If KeyDown(KEY_D) Then TurnEntity(face1.Entity(), 0, -5, 0)
	If KeyDown(KEY_S) Then MoveEntity(face1.Entity(), 0, 0 ,-1)

	faceeach2(face1, face2)
	faceeach2(face2, face1)
	
RenderWorld
Flip
Wend

End 

Type face
	Global faces:TList = New TList
	
	Field x:Float
	Field y:Float
	Field rot:Float = 0
	Field radius:Float = 50
	Field anglimit:Float = 45
	
	Field face_mesh:TEntity
	Field face_front:TEntity
	
	Function Create:face()
		Local new_face:face = New face
		new_face.Init()
		Return new_face
	EndFunction

	Method Entity:TEntity()
		Return face_mesh
	EndMethod

	Method New()
		faces.AddFirst(Self)
	End Method

	Method Init()
		face_mesh = CreateCube()
		face_front = CreateSphere()
		ScaleEntity(face_front, .5, .5, .5)
		PositionEntity(face_front, 0, 0, 3)
		EntityParent(face_front, face_mesh)
	EndMethod
	
End Type

Function faceeach(f1:face, f2:face)
	Local ca:Float = EntityYaw(f1.Entity(), True)
	Local ta:Float = ATan2(EntityZ(f2.Entity(), True) - EntityZ(f1.Entity(), True), EntityX(f2.Entity(), True) - EntityX(f1.Entity(), True))
	
	Local la:Float = 45
	Local da:Float = ta - ca
	Local dif:Float = Abs(da) - la
	
	If dif > 0 Then
		Local da1:Float = Cos(da + la)
		Local da2:Float = Cos(da - la)
		If da1 >= da2 Then
			RotateEntity(f1.Entity(), 0, ta + la, 0, True)
		Else
			RotateEntity(f1.Entity(), 0, ta - la, 0, True)
		End If
	EndIf
End Function

Function faceeach2(f1:face, f2:face)
		Local angle_limit:Int = 45
		Local current_rot:Float = EntityYaw(f1.Entity(), True)
		Local current_rot_x:Float = EntityPitch(f1.Entity(), True)

		' find the rotation to the other boxer
		PointEntity(f1.Entity(), f2.Entity())		
		Local final_rot:Float = EntityYaw(f1.Entity(), True)
				
		Local current_angle:Float = 0
		Local adjust_angle:Float = 0

		' check if we crossed the 180 border
		If (current_rot < -90 And current_rot > -179.99999 And final_rot > 0) Or (current_rot > 90 And current_rot < 179.99999  And final_rot < 0)
			
			current_angle = (180 - Abs(current_rot)) + (180 - Abs(final_rot))

			If (current_angle) > angle_limit Then  
				' turn left here 
				adjust_angle = (Abs(current_angle) - angle_limit)
				If current_rot < 0 Then adjust_angle :* -1
			EndIf
		Else 
			current_angle = current_rot - final_rot	
			If Abs(current_angle) > angle_limit Then
				adjust_angle = (Abs(current_angle) - angle_limit)
				If current_angle > 0 Then adjust_angle :* -1
			EndIf
		EndIf
				
		' restore original rotation
		RotateEntity(f1.Entity(), current_rot_x, current_rot, 0, True)
				
		' adjust to within bounds
		If adjust_angle <> 0 Then TurnEntity(f1.Entity(), 0, adjust_angle, 0, True)

EndFunction