Code archives/Miscellaneous/Blitz Lexical Analyser

This code has been declared by its author to be Public Domain code.

Download source code

Blitz Lexical Analyser by Steve Hill
(Posted 23 years ago)
This code splits a Blitz source file into tokens.

Each token represents items like operators, keywords, comments, end of line, functions etc.

Useful if you want to write a pre-processor or other tool that manipulates Blitz code files.

Contains an example for just printing the tokens in a file. 
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;
; Lexer.bb
;
; Tokenises Blitz Basic code
;
; Steve Hill, 2003
;
; OpenState(fileName$) - creates a new TState
; CloseState()         - destroys and closes the current state
; GetToken(state)      - read the next token
;
; The current token is available in state\tok$
;
; Versions
; 0.1			Initial version		27 Aug 2003
; 0.2			Added >< => =<		29 Aug 2003
;

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; CONSTANTS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

; Constants for various character types
;
Const  SPACE        = 32
Const  TAB			= 9
Const  CR			= 13
Const  LF           = 10

Global WHITE_SPACE$ = Chr$(SPACE) + Chr$(TAB) + Chr$(CR)
Const  ALPHA$       = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
Const  DIGITS$      = "0123456789"
Const  HEXDIGITS$   = "0123456789abcdefABCDEF"
Const  BINDIGITS$   = "01"
Const  DELIM$		= "^*+-~<>/\#%.$()[],=\:"
Global QUOTE$		= Chr$(34)
Global ALPHANUM$	= ALPHA$ + DIGITS$

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; TYPES
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

; TState
;
; Keeps track of the current file, look-ahead character and token.
; Can be used as a stack for include files.
;
Type TState
	Field file
	Field ch$
	Field tok$
	Field tokType
	Field lineNum
	; Field charNum
End Type

Const TOK_WORD			= 0
Const TOK_OPERATOR		= 1
Const TOK_COMMENT		= 2
Const TOK_DEC_NUMBER	= 3
Const TOK_HEX_NUMBER	= 4
Const TOK_BIN_NUMBER	= 5
Const TOK_STRING		= 6
Const TOK_EOL			= 7
Const TOK_EOF			= 8
Const TOK_UNKNOWN		= 9

; TDescriptor
;
; Describes a function: name, return type and parameter type list.
; Assigned unique id for each functions ... its "pointer"
;
Type TDescriptor
	Field name$
	Field typ$
	Field params$
	Field id
End Type

; Error
;
; Something has gone wrong bail out.
;
Function Error(e$, state.TState)
	Print e$
	If state\file <> 0
		Print "Error on line " + Str$(state\lineNum)
	EndIf
	Print "Press a key"
	WaitKey
	End
End Function

; OpenState
;
; Open the file initialise the fields
;
Function OpenState(name$)
	Print "Parsing " + name$
	state.TState = New TState
	state\file = ReadFile(name$)
	If state\file = 0 Then
		Error("File " + name$ + "not found", state)
	EndIf
	state\lineNum = 1
	state\ch$ = ""
	state\tok$ = ""
	
	GetToken(state)
End Function

; CloseState
;
; Close current file, pop state
;
Function CloseState()
	For state.TState = Each TState
		CloseFile(state\file)
	Next
	
	Delete Each TState
End Function

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; LEXICAL FUNCTIONS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

; GetChar
;
; Read a character from the current file into the state
;
Function GetChar(state.TState)
	If state\ch$ = Chr$(LF)
		state\lineNum = state\LineNum + 1
	EndIf

	If Eof(state\file) Then
		state\ch$ = ""
		Return
	EndIf
	
	state\ch$ = Chr$(ReadByte(state\file))	
End Function

; SkipSpace
;
; Skip white space
;
Function SkipSpace(state.TState)
	While Instr(WHITE_SPACE$, state\ch$) <> 0
		GetChar(state)
		If state\ch$ = "" Return
	Wend
End Function

; GetFollowing
;
; Generic token reader, reads while characters are
; those in pat.  Places token in state.
;
Function GetFollowing(state.TState, pat$)
	tok$ = state\ch$
	GetChar(state)
	While Instr(pat$, state\ch$) <> 0
		If state\ch$ = "" Exit
		tok$ = tok$ + state\ch$
		GetChar(state)
	Wend
	state\tok$ = tok$
End Function

; GetDecNumber
;
; eg. 1 or 1.2
;
Function GetDecNumber$(state.TState)
	GetFollowing(state, DIGITS$ + ".")
	state\tokType = TOK_DEC_NUMBER
End Function

; GetBinNumber
;
; eg. %1100
;
Function GetBinNumber$(state.TState)
	GetFollowing(state, BINDIGITS$)
	state\tokType = TOK_BIN_NUMBER
End Function

; GetHexNumber
;
; eg. $abC1
;
Function GetHexNumber$(state.TState)
	GetFollowing(state, HEXDIGITS$)
	If state\tok$ = "$"
		state\tokType = TOK_OPERATOR
	Else
		state\tokType = TOK_HEX_NUMBER
	EndIf
End Function

; GetEOL
;
; Get end of line
Function GetEOL(state.TState)
	state\tok$ = state\ch$
	GetChar(state)
	state\tokType = TOK_EOL
End Function

; GetWord
;
; eg. myVar_2 or WaitKey
;
Function GetWord(state.TState)
	GetFollowing(state, ALPHANUM$ + "_")
	If state\tok$ = "Or" Or state\tok$ = "And" Or state\tok$ = "Xor" Then
		state\tokType = TOK_OPERATOR
	Else
		state\tokType = TOK_WORD
	EndIf
End Function

; GetOperator
;
; eg. , . \ + - = > < <> etc.
;
Function GetOperator(state.TState)
	t$ = state\ch$
	GetChar(state)

	r$ = t$
	
	Select t$
	Case ">"
		t$ = state\ch$
		Select t$ 
		Case "="
			r$ = ">="
			GetChar(state)
		Case "<"
			r$ = "><"
			GetChar(state)
		Default
			r$ = ">"
		End Select
	Case "<"
		t$ = state\ch$
		Select t$
		Case "="
			r$ = "<="
			GetChar(state)
		Case ">"
			r$ = "<>"
			GetChar(state)
		Default
			r$ = "<"
		End Select
	Case "="
		t$ = state\ch$
		Select t$
		Case ">"
			r$ = "=>"
			GetChar(state)
		Case "<"
			r$ = "=<"
			GetChar(state)
		Default
			r$ = "="
		End Select
	End Select
		
	state\tok$ = r$
	state\tokType = TOK_OPERATOR
End Function

; GetComment
;
; eg. ; a comment
;
Function GetComment(state.TState)
	tok$ = state\ch$
	GetChar(state)
	While state\ch$ <> Chr$(LF)
		If state\ch$ = "" Exit
		If state\ch$ <> Chr$(CR)
			tok$ = tok$ + state\ch$
		EndIf
		GetChar(state)
	Wend
	state\tok$ = tok$
	state\tokType = TOK_COMMENT
End Function

; GetString
;
; eg. "a string"
;
Function GetString(state.TState)
	tok$ = ""
	GetChar(state)
	While state\ch$ <> QUOTE$ And state\ch$ <> ""
		tok$ = tok$ + state\ch$
		GetChar(state)
	Wend

	If state\ch$ <> ""
		state\tok$ = QUOTE$ + tok$ + QUOTE$
		GetChar(state)
	EndIf
	
	state\tokType = TOK_STRING
End Function

; GetToken
;
; Use first character to determine type of token and then
; read appropriate token using the corresponding Get function
;
Function GetToken(state.TState)
	SkipSpace(state)

	ch$ = state\ch$
	
	If ch$ = "" Then
		state\tok$ = ""
		state\tokType = TOK_EOF
		Return
	EndIf
	
	If Instr(DIGITS$, ch$) <> 0 Then
		GetDecNumber$(state)
	ElseIf Instr(ALPHA$, ch$) <> 0 Then
		GetWord(state)
	ElseIf ch$ = ";" Then
		GetComment(state)
	ElseIf ch$ = QUOTE$ Then
		GetString(state)
	ElseIf ch$ = "%" Then
		GetBinNumber(state)
	ElseIf ch$ = "$" Then
		GetHexNumber(state)
	ElseIf Instr(DELIM$, ch$) <> 0 Then
		GetOperator(state)
	ElseIf ch$ = Chr$(LF)
		GetEOL(state)
	Else
		Error("Unrecognised character " + ch$ + "(" + Asc(ch$) + ") in file", state)
	EndIf

	; DebugLog Str$(state\lineNum) + ": " + state\tok$
	
End Function

; Example usage
;
;
;
;inFile$  = Input$("Input file: ")
;
;OpenState(inFile$)
;state.TState = Last TState
;While state\tok$ <> ""
;	If state\tokType <> TOK_EOL Then
;		Print state\tok$
;	EndIf
;	GetToken(state)
;Wend
;CloseState()
;
;Print "Press a key"
;WaitKey
;
;End

Comments

_33
(Posted 20 years ago)
My practicle adaptation of previous code.

func_tokenizer.bb :
; Author: Steve Hill
; Date: 2003-08-27 13:41:23
; Title: Blitz Lexical Analyser
; Description: Blitz functions to split a Blitz source file into tokens
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; OpenState(file$,mode) - creates a new TState
; GetToken(state)       - read the next token
; CloseState()          - destroys and closes the current state
;
; The current token is available in state\tok$
;
; Versions
; 0.1	Initial version                                   27 Aug 2003
; 0.2	Added >< => =<	                                  29 Aug 2003
; 0.3   Added "mode" select                               16 Apr 2007 by _33
;       REPLACED "tok" by "token" (this is not Tokamak)   16 Apr 2007 by _33
; 0.4   Removed PRINTs and ENDs which made this unusable  16 Apr 2007 by _33
;       Added some error management "ok flag"             16 Apr 2007 by _33
;
; FUTURE RELEASE: -Add function to return FLOAT, INT, and STRING
;                 -More cleanup, optimize, expand on concept
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

Const  SPACE        = 32
Const  TAB			= 9
Const  CR			= 13
Const  LF           = 10

Global WHITE_SPACE$   = Chr$(SPACE) + Chr$(TAB) + Chr$(CR)
Const  ALPHA$         = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
Const  DIGITS$        = "0123456789"
Const  HEXDIGITS$     = "0123456789abcdefABCDEF"
Const  BINDIGITS$     = "01"
Const  DELIM$		  = "^*+-~<>/\#%.$()[],=\:?!"
Global QUOTE$         = Chr$(34)
Global ALPHANUM$      = ALPHA$ + DIGITS$

; TState
;
; Keeps track of the current file, look-ahead character and token.
; Can be used as a stack for include files.
;
Type TState
	Field file
	Field mode
	Field ch$
	Field token$
	Field tokenType
	Field lineNum
	; Field charNum
	Field content$     ;v0.3
	Field content_len% ;v0.3
    Field ch_pos       ;v0.3
End Type


Const TOKEN_WORD		= 0
Const TOKEN_OPERATOR	= 1
Const TOKEN_COMMENT		= 2
Const TOKEN_DEC_NUMBER	= 3
Const TOKEN_HEX_NUMBER	= 4
Const TOKEN_BIN_NUMBER	= 5
Const TOKEN_STRING		= 6
Const TOKEN_EOL			= 7
Const TOKEN_EOF			= 8
Const TOKEN_UNKNOWN		= 9

; TDescriptor
;
; Describes a function: name, return type and parameter type list.
; Assigned unique id for each functions ... its "pointer"
;
Type TDescriptor
	Field name$
	Field typ$
	Field params$
	Field id
End Type

; OpenState
;
; Open the file initialise the fields
; 1 = file, 2 = inline tokenize
Function OpenState(fi$,mode)
   state.TState = New TState
   state\mode = mode
   state\ch_pos = 0
   If state\mode = 1 Then
      state\file = ReadFile(fi$)
      If state\file = 0 Then
         Return 0
      EndIf
   ElseIf state\mode = 2
      state\content$ = fi$
   EndIf
   state\lineNum = 1
   state\ch$ = ""
   state\token$ = ""
   Return GetToken(state)
End Function

; CloseState
;
; Close current file, pop state
;
Function CloseState()
   For state.TState = Each TState
      If state\mode = 1 Then CloseFile(state\file)
   Next
   Delete Each TState
End Function

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; LEXICAL FUNCTIONS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

; GetChar
;
; Read a character from the current file into the state
;
Function GetChar(state.TState)
   If state\ch$ = Chr$(LF) Then state\lineNum = state\LineNum + 1 : state\ch_pos = 0

   If state\mode = 1 Then
      If Eof(state\file) Then state\ch$ = "" : Return
      state\ch$ = Chr$(ReadByte(state\file))	
   Else
      state\ch_pos = state\ch_pos + 1
      state\ch$ = Mid$(state\content$,state\ch_pos,1)
   EndIf
;morel
End Function

; SkipSpace
;
; Skip white space
;
Function SkipSpace(state.TState)
	While Instr(WHITE_SPACE$, state\ch$) <> 0
		GetChar(state)
		If state\ch$ = "" Return
	Wend
End Function

; GetFollowing
;
; Generic token reader, reads while characters are
; those in pat.  Places token in state.
;
Function GetFollowing(state.TState, pat$)
	token$ = state\ch$
	GetChar(state)
	While Instr(pat$, state\ch$) <> 0
		If state\ch$ = "" Exit
		token$ = token$ + state\ch$
		GetChar(state)
	Wend
	state\token$ = token$
End Function

; GetDecNumber
;
; eg. 1 or 1.2
Function GetDecNumber$(state.TState)
	GetFollowing(state, DIGITS$ + ".")
	state\tokenType = TOKEN_DEC_NUMBER
End Function

; GetBinNumber
;
; eg. %1100
Function GetBinNumber$(state.TState)
	GetFollowing(state, BINDIGITS$)
	state\tokenType = TOKEN_BIN_NUMBER
End Function

; GetHexNumber
;
; eg. $abC1
Function GetHexNumber$(state.TState)
	GetFollowing(state, HEXDIGITS$)
	If state\token$ = "$"
		state\tokenType = TOKEN_OPERATOR
	Else
		state\tokenType = TOKEN_HEX_NUMBER
	EndIf
End Function

; GetEOL
;
; Get end of line
Function GetEOL(state.TState)
	state\token$ = state\ch$
	GetChar(state)
	state\tokenType = TOKEN_EOL
End Function

; GetWord
;
; eg. myVar_2 or WaitKey
Function GetWord(state.TState)
	GetFollowing(state, ALPHANUM$ + "_")
	If state\token$ = "Or" Or state\token$ = "And" Or state\token$ = "Xor" Then
		state\tokenType = TOKEN_OPERATOR
	Else
		state\tokenType = TOKEN_WORD
	EndIf
End Function

Function GetOperator(state.TState)
	t$ = state\ch$
	GetChar(state)

	r$ = t$
	
	Select t$
	Case ">"
		t$ = state\ch$
		Select t$ 
		Case "="
			r$ = ">="
			GetChar(state)
		Case "<"
			r$ = "><"
			GetChar(state)
		Default
			r$ = ">"
		End Select
	Case "<"
		t$ = state\ch$
		Select t$
		Case "="
			r$ = "<="
			GetChar(state)
		Case ">"
			r$ = "<>"
			GetChar(state)
		Default
			r$ = "<"
		End Select
	Case "="
		t$ = state\ch$
		Select t$
		Case ">"
			r$ = "=>"
			GetChar(state)
		Case "<"
			r$ = "=<"
			GetChar(state)
		Default
			r$ = "="
		End Select
	End Select
		
	state\token$ = r$
	state\tokenType = TOKEN_OPERATOR
End Function

Function GetComment(state.TState)
   token$ = state\ch$
   GetChar(state)
   While state\ch$ <> Chr$(LF)
      If state\ch$ = "" Exit
      If state\ch$ <> Chr$(CR)
         token$ = token$ + state\ch$
      EndIf
      GetChar(state)
   Wend
   state\token$ = token$
   state\tokenType = TOKEN_COMMENT
End Function

Function GetString(state.TState)
   state\token$ = ""
   GetChar(state)
   While state\ch$ <> QUOTE$ And state\ch$ <> ""
      state\token$ = state\token$ + state\ch$
      GetChar(state)
   Wend
   If state\ch$ <> ""
      GetChar(state)
   EndIf
   state\tokenType = TOKEN_STRING
End Function

Function GetUnrecognized(state.TState)
   state\token$ = state\ch$
   state\tokenType = TOKEN_UNKNOWN
   GetChar(state)
End Function


; GetToken
;
; Use first character to determine type of token and then
; read appropriate token using the corresponding Get function
Function GetToken%(state.TState)
   SkipSpace(state)
   ch$ = state\ch$
   If ch$ = "" Then
      state\token$ = ""
      state\tokenType = TOKEN_EOF
      Return 1
   EndIf
   If Instr(DIGITS$, ch$) <> 0 Then
      GetDecNumber$(state)
   ElseIf Instr(ALPHA$, ch$) <> 0 Then
      GetWord(state)
   ElseIf ch$ = ";" Then
      GetComment(state)
   ElseIf ch$ = QUOTE$ Then
      GetString(state)
   ElseIf ch$ = "%" Then
      GetBinNumber(state)
   ElseIf ch$ = "$" Then
      GetHexNumber(state)
   ElseIf Instr(DELIM$, ch$) <> 0 Then
      GetOperator(state)
   ElseIf ch$ = Chr$(LF)
      GetEOL(state)
   Else
      GetUnrecognized(state)
   EndIf
   Return 1
End Function


TEST:
Graphics 1024,768,32,2
Include "func_tokenizer.bb"

mode = Input$("type 1 to read from file, 2 to enter your own test: ")
inFile$  = Input$("Input file: ")
While inFile$ <>""
   Print ""
   Print "Parsing " + inFile$

   ok = OpenState(inFile$, mode)
   state.TState = Last TState
    While ok
      Color 0,255,0
      Select state\tokenType
      Case TOKEN_WORD       Write "W" : Color 0,255,255 : Write state\token$
      Case TOKEN_OPERATOR   Write "O" : Color 0,255,255 : Write state\token$
      Case TOKEN_COMMENT    Write "C" : Color 0,255,255 : Write state\token$
      Case TOKEN_DEC_NUMBER Write "D" : Color 0,255,255 : Write state\token$
      Case TOKEN_HEX_NUMBER Write "H" : Color 0,255,255 : Write state\token$
      Case TOKEN_BIN_NUMBER Write "B" : Color 0,255,255 : Write state\token$
      Case TOKEN_STRING     Write "S" : Color 255,255,255 : Write state\token$
      Case TOKEN_EOL        Color 255,255,0 : Write "EOL" : Print ""
      Case TOKEN_EOF        Color 255,0,255 : Write "EOF" : Print "" : ok = 0
      Case TOKEN_UNKNOWN    Color 255,0,0 : Write "U"
      End Select
      If ok Then ok = GetToken(state)
      If KeyHit(1) Then ok = 0
   Wend
   CloseState()
   Color 255,255,255
   inFile$  = Input$("Input file: ")
Wend
End


Code Archives Forum