python实现词法分析器


Posted in Python onJanuary 31, 2019

简单Python词法分析器实现,供大家参考,具体内容如下

词法分析器状态转换图:

python实现词法分析器

词法分析器总流程图:

python实现词法分析器

预处理程序:

python实现词法分析器

词法分析器:

python实现词法分析器

词法分析器程序详细设计

python实现词法分析器

详细代码实现:

#!/usr/bin/env python3.4
# coding=utf-8
import sys
import string


keywards = {}

# 关键字部分
keywards['False'] = 101
keywards['class'] = 102
keywards['finally'] = 103
keywards['is'] = 104
keywards['return'] = 105
keywards['None'] = 106
keywards['continue'] = 107
keywards['for'] = 108
keywards['lambda'] = 109
keywards['try'] = 110
keywards['True'] = 111
keywards['def'] = 112
keywards['from'] = 113
keywards['nonlocal'] = 114
keywards['while'] = 115
keywards['and'] = 116
keywards['del'] = 117
keywards['global'] = 118
keywards['not'] = 119
keywards['with'] = 120
keywards['as'] = 121
keywards['elif'] = 122
keywards['if'] = 123
keywards['or'] = 124
keywards['yield'] = 125
keywards['assert'] = 126
keywards['else'] = 127
keywards['import'] = 128
keywards['pass'] = 129
keywards['break'] = 130
keywards['except'] = 131
keywards['in'] = 132
keywards['raise'] = 133

# 符号
keywards['+'] = 201
keywards['-'] = 202
keywards['*'] = 203
keywards['/'] = 204
keywards['='] = 205
keywards[':'] = 206
keywards['<'] = 207
keywards['>'] = 208
keywards['%'] = 209
keywards['&'] = 210
keywards['!'] = 211
keywards['('] = 212
keywards[')'] = 213
keywards['['] = 214
keywards[']'] = 215
keywards['{'] = 216
keywards['}'] = 217
keywards['#'] = 218
keywards['|'] = 219
keywards[','] = 220
# 变量
# keywards['var'] = 301

# 常量
# keywards['const'] = 401

# Error
# keywards['const'] = 501

signlist = {}


# 预处理函数,将文件中的空格,换行等无关字符处理掉
def pretreatment(file_name):
 try:
  fp_read = open(file_name, 'r')
  fp_write = open('file.tmp', 'w')
  sign = 0
  while True:
   read = fp_read.readline()
   if not read:
    break
   length = len(read)
   i = -1
   while i < length - 1:
    i += 1
    if sign == 0:
     if read[i] == ' ':
      continue
    if read[i] == '#':
     break
    elif read[i] == ' ':
     if sign == 1:
      continue
     else:
      sign = 1
      fp_write.write(' ')
    elif read[i] == '\t':
     if sign == 1:
      continue
     else:
      sign = 1
      fp_write.write(' ')
    elif read[i] == '\n':
     if sign == 1:
      continue
     else:
      fp_write.write(' ')
      sign = 1
    elif read[i] == '"':
     fp_write.write(read[i])
     i += 1
     while i < length and read[i] != '"':
      fp_write.write(read[i])
      i += 1
     if i >= length:
      break
     fp_write.write(read[i])
    elif read[i] == "'":
     fp_write.write(read[i])
     i += 1
     while i < length and read[i] != "'":
      fp_write.write(read[i])
      i += 1
     if i >= length:
      break
     fp_write.write(read[i])
    else:
     sign = 3
     fp_write.write(read[i])
 except Exception:
  print(file_name, ': This FileName Not Found!')


def save(string):
 if string in keywards.keys():
  if string not in signlist.keys():
   signlist[string] = keywards[string]
 else:
  try:
   float(string)
   save_const(string)
  except ValueError:
   save_var(string)


def save_var(string):
 if string not in signlist.keys():
  if len(string.strip()) < 1:
   pass
  else:
   if is_signal(string) == 1:
    signlist[string] = 301
   else:
    signlist[string] = 501


def save_const(string):
 if string not in signlist.keys():
  signlist[string] = 401


def save_error(string):
 if string not in signlist.keys():
  signlist[string] = 501


def is_signal(s):
 if s[0] == '_' or s[0] in string.ascii_letters:
  for i in s:
   if i in string.ascii_letters or i == '_' or i in string.digits:
    pass
   else:
    return 0
  return 1
 else:
  return 0


def recognition(filename):
 try:
  fp_read = open(filename, 'r')
  string = ""
  sign = 0
  while True:
   read = fp_read.read(1)
   if not read:
    break

   if read == ' ':
    if len(string.strip()) < 1:
     sign = 0
     pass
    else:
     if sign == 1 or sign == 2:
      string += read
     else:
      save(string)
      string = ""
      sign = 0
   elif read == '(':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('(')
   elif read == ')':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(')')
   elif read == '[':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('[')
   elif read == ']':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(']')
   elif read == '{':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('{')
   elif read == '}':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('}')
   elif read == '<':
    save(string)
    string = ""
    save('<')
   elif read == '>':
    save(string)
    string = ""
    save('>')
   elif read == ',':
    save(string)
    string = ""
    save(',')
   elif read == "'":
    string += read
    if sign == 1:
     sign = 0
     save_const(string)
     string = ""
    else:
     if sign != 2:
      sign = 1
   elif read == '"':
    string += read
    if sign == 2:
     sign = 0
     save_const(string)
     string = ""
    else:
     if sign != 1:
      sign = 2
   elif read == ':':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(':')
   elif read == '+':
    save(string)
    string = ""
    save('+')
   elif read == '=':
    save(string)
    string = ""
    save('=')
   else:
    string += read

 except Exception as e:
  print(e)


def main():
 if len(sys.argv) < 2:
  print("Please Input FileName")
 else:
  pretreatment(sys.argv[1])
 recognition('file.tmp')
 for i in signlist.keys():
  print("(", signlist[i], ",", i, ")")


if __name__ == '__main__':
 main()

以上就是本文的全部内容,希望对大家的学习有所帮助,也希望大家多多支持三水点靠木。

Python 相关文章推荐
简单上手Python中装饰器的使用
Jul 12 Python
pandas全表查询定位某个值所在行列的方法
Apr 12 Python
python实现Flappy Bird源码
Dec 24 Python
图文详解python安装Scrapy框架步骤
May 20 Python
Python3基础教程之递归函数简单示例
Jun 07 Python
如何运行.ipynb文件的图文讲解
Jun 27 Python
Python read函数按字节(字符)读取文件的实现
Jul 03 Python
Python实现K折交叉验证法的方法步骤
Jul 11 Python
Python连接Oracle之环境配置、实例代码及报错解决方法详解
Feb 11 Python
Python netmiko模块的使用
Feb 14 Python
keras 回调函数Callbacks 断点ModelCheckpoint教程
Jun 18 Python
Python实现王者荣耀自动刷金币的完整步骤
Jan 22 Python
使用Python 统计高频字数的方法
Jan 31 #Python
python3正则提取字符串里的中文实例
Jan 31 #Python
python统计中文字符数量的两种方法
Jan 31 #Python
解决python写入带有中文的字符到文件错误的问题
Jan 31 #Python
在Python中居然可以定义两个同名通参数的函数
Jan 31 #Python
Python的条件表达式和lambda表达式实例
Jan 31 #Python
OpenCV+Python识别车牌和字符分割的实现
Jan 31 #Python
You might like
Yii框架数据模型的验证规则rules()被执行的方法
2016/12/02 PHP
PHP安装memcache扩展的步骤讲解
2019/02/14 PHP
tp5框架前台无限极导航菜单类实现方法分析
2020/03/29 PHP
基于Jquery的实现回车键Enter切换焦点
2010/09/14 Javascript
javascript中文本框中输入法切换的问题
2013/12/10 Javascript
JavaScript中的函数的两种定义方式和函数变量赋值
2014/05/12 Javascript
jquery获取css的color值返回RGB的方法
2015/12/18 Javascript
jquery拖拽效果完整实例(附demo源码下载)
2016/01/14 Javascript
详解JavaScript的另类写法
2016/04/11 Javascript
js获取隐藏元素宽高的实现方法
2016/05/19 Javascript
Bootstrap编写一个同时适用于PC、平板、手机的登陆页面
2016/06/30 Javascript
ajax级联菜单实现方法实例分析
2016/11/28 Javascript
js实现日历的简单算法
2017/01/24 Javascript
AngularJS实现页面跳转后自动弹出对话框实例代码
2017/08/02 Javascript
JS获取一个表单字段中多条数据并转化为json格式
2017/10/17 Javascript
vue中的scope使用详解
2017/10/29 Javascript
JS严格模式知识点总结
2018/02/27 Javascript
Node.js Event Loop各阶段讲解
2019/03/08 Javascript
python中的sort方法使用详解
2014/07/25 Python
python实现将英文单词表示的数字转换成阿拉伯数字的方法
2015/07/02 Python
Python机器学习之决策树算法实例详解
2017/12/06 Python
对python .txt文件读取及数据处理方法总结
2018/04/23 Python
Python3.5 Pandas模块之DataFrame用法实例分析
2019/04/23 Python
python 循环数据赋值实例
2019/12/02 Python
python 读写文件包含多种编码格式的解决方式
2019/12/20 Python
Win10下配置tensorflow-gpu的详细教程(无VS2015/2017)
2020/07/14 Python
Python 排序最长英文单词链(列表中前一个单词末字母是下一个单词的首字母)
2020/12/14 Python
CSS3五个技巧给你的网站带来出色的效果
2009/04/02 HTML / CSS
CSS3的Border-radius轻松制作圆角
2012/12/24 HTML / CSS
触摸春天教学反思
2014/02/03 职场文书
营业用房租赁协议书
2014/11/26 职场文书
2015年青年教师工作总结
2015/05/25 职场文书
美容院员工规章制度
2015/08/05 职场文书
js实现自动锁屏功能
2021/06/02 Javascript
Django模型层实现多表关系创建和多表操作
2021/07/21 Python
WebRTC记录音视频流(web技术分享)
2022/02/24 Javascript