python实现词法分析

#请先安装Ply

# -*- coding: utf-8 -*-
#--------------------------------------------------------------------------
#Author:Jmdebugger
#email: pengkailb@gmail.com
#date: 2013-9-17
#--------------------------------------------------------------------------
import ply.lex as lex

tokens = [
"TOKEN_IDENT",
"TOKEN_INT",
"TOKEN_FLOAT",
"TOKEN_STRING",
"TOKEN_OP",
"TOKEN_DELIM_COMMA",        #,
"TOKEN_DELIM_OPEN_PAREN",   #(
"TOKEN_DELIM_CLOSE_PAREN",  #)
"TOKEN_DELIM_OPEN_BRACKET", #[
"TOKEN_DELIM_CLOSE_BRACKET",#]
"TOKEN_DELIM_OPEN_BRACE",   #{
"TOKEN_DELIM_CLOSE_BRACE",  #}
"TOKEN_DELIM_SEMICOLON"    #;

]

reserved = {
'if'     :   'TOKEN_RSRVD_IF',
'else'   :   'TOKEN_RSRVD_ELSE',
'true'   :   'TOKEN_RSRVD_TRUE',
'false'  :   'TOKEN_RSRVD_FALSE',
'while'  :   'TOKEN_RSRVD_WHILE',
'break'  :   'TOKEN_RSRVD_BREAK',
'continue':  'TOKEN_RSRVD_CONTINUE',
'goto'   :   'TOKEN_RSRVD_GOTO',
'func'   :   'TOKEN_RSRVD_FUNC',
'var'    :   'TOKEN_RSRVD_VAR',
'for'    :   'TOKEN_RSRVD_FOR',
'return' :   'TOKEN_RSRVD_RETURN'
}

tokens += reserved .values()

t_ignore = r' \t\r'
def t_COMMENT(t):
r'(/\*(.|\n)*?\*/)|(\/\/.*)'
pass

def t_newline(t):
r'\n+'
t.lexer.lineno += len(t.value)

def t_error(t):
print "LaunchScript error: "+repr(t.value)

def t_TOKEN_IDENT(t):
r'[a-zA-Z_][a-zA-Z_0-9]*' #标识符
t.type = reserved.get(t.value , 'TOKEN_IDENT')
return t

def t_TOKEN_INT(t):
r'(0x[a-fA-F0-9]+)|([0-9]+)'
return t

t_TOKEN_FLOAT =  r'[0-9]*\.[0-9]+'
t_TOKEN_STRING = r'(\"([^\\\r]|(\\.))*?\")' #|(\"([^\\\n]|(\\.))*?\")' only for windows
t_TOKEN_DELIM_COMMA = r'\,'
t_TOKEN_DELIM_OPEN_PAREN = r'\('
t_TOKEN_DELIM_CLOSE_PAREN = r'\)'
t_TOKEN_DELIM_OPEN_BRACKET = r'\['
t_TOKEN_DELIM_CLOSE_BRACKET = r'\]'
t_TOKEN_DELIM_OPEN_BRACE = r'\{'
t_TOKEN_DELIM_CLOSE_BRACE = r'\}'
t_TOKEN_DELIM_SEMICOLON = r'\;'

def t_TOKEN_OP(t):
r'(\<\<\=)|(\>\>\=)|([\+\-\*\/\%\&\|\^\=\!\>\<]\=)|(\|\|)|(\&\&)|(\+\+)|(\-\-)|[\+\-\*\/\%\^\=\&\|\>\<\!\~]'
return t

if __name__ == "__main__":
lexer = lex.lex()
f = open("./test.txt" , 'rb')
data = f.read()
f.close()
lexer.input(data)

while True:
tok = lexer.token()
if not tok: break      # No more input
print tok.value+"\t---->\t"+tok.type