python实现词法分析器


Posted in Python onJanuary 31, 2019

简单Python词法分析器实现,供大家参考,具体内容如下

词法分析器状态转换图:

python实现词法分析器

词法分析器总流程图:

python实现词法分析器

预处理程序:

python实现词法分析器

词法分析器:

python实现词法分析器

词法分析器程序详细设计

python实现词法分析器

详细代码实现:

#!/usr/bin/env python3.4
# coding=utf-8
import sys
import string


keywards = {}

# 关键字部分
keywards['False'] = 101
keywards['class'] = 102
keywards['finally'] = 103
keywards['is'] = 104
keywards['return'] = 105
keywards['None'] = 106
keywards['continue'] = 107
keywards['for'] = 108
keywards['lambda'] = 109
keywards['try'] = 110
keywards['True'] = 111
keywards['def'] = 112
keywards['from'] = 113
keywards['nonlocal'] = 114
keywards['while'] = 115
keywards['and'] = 116
keywards['del'] = 117
keywards['global'] = 118
keywards['not'] = 119
keywards['with'] = 120
keywards['as'] = 121
keywards['elif'] = 122
keywards['if'] = 123
keywards['or'] = 124
keywards['yield'] = 125
keywards['assert'] = 126
keywards['else'] = 127
keywards['import'] = 128
keywards['pass'] = 129
keywards['break'] = 130
keywards['except'] = 131
keywards['in'] = 132
keywards['raise'] = 133

# 符号
keywards['+'] = 201
keywards['-'] = 202
keywards['*'] = 203
keywards['/'] = 204
keywards['='] = 205
keywards[':'] = 206
keywards['<'] = 207
keywards['>'] = 208
keywards['%'] = 209
keywards['&'] = 210
keywards['!'] = 211
keywards['('] = 212
keywards[')'] = 213
keywards['['] = 214
keywards[']'] = 215
keywards['{'] = 216
keywards['}'] = 217
keywards['#'] = 218
keywards['|'] = 219
keywards[','] = 220
# 变量
# keywards['var'] = 301

# 常量
# keywards['const'] = 401

# Error
# keywards['const'] = 501

signlist = {}


# 预处理函数,将文件中的空格,换行等无关字符处理掉
def pretreatment(file_name):
 try:
  fp_read = open(file_name, 'r')
  fp_write = open('file.tmp', 'w')
  sign = 0
  while True:
   read = fp_read.readline()
   if not read:
    break
   length = len(read)
   i = -1
   while i < length - 1:
    i += 1
    if sign == 0:
     if read[i] == ' ':
      continue
    if read[i] == '#':
     break
    elif read[i] == ' ':
     if sign == 1:
      continue
     else:
      sign = 1
      fp_write.write(' ')
    elif read[i] == '\t':
     if sign == 1:
      continue
     else:
      sign = 1
      fp_write.write(' ')
    elif read[i] == '\n':
     if sign == 1:
      continue
     else:
      fp_write.write(' ')
      sign = 1
    elif read[i] == '"':
     fp_write.write(read[i])
     i += 1
     while i < length and read[i] != '"':
      fp_write.write(read[i])
      i += 1
     if i >= length:
      break
     fp_write.write(read[i])
    elif read[i] == "'":
     fp_write.write(read[i])
     i += 1
     while i < length and read[i] != "'":
      fp_write.write(read[i])
      i += 1
     if i >= length:
      break
     fp_write.write(read[i])
    else:
     sign = 3
     fp_write.write(read[i])
 except Exception:
  print(file_name, ': This FileName Not Found!')


def save(string):
 if string in keywards.keys():
  if string not in signlist.keys():
   signlist[string] = keywards[string]
 else:
  try:
   float(string)
   save_const(string)
  except ValueError:
   save_var(string)


def save_var(string):
 if string not in signlist.keys():
  if len(string.strip()) < 1:
   pass
  else:
   if is_signal(string) == 1:
    signlist[string] = 301
   else:
    signlist[string] = 501


def save_const(string):
 if string not in signlist.keys():
  signlist[string] = 401


def save_error(string):
 if string not in signlist.keys():
  signlist[string] = 501


def is_signal(s):
 if s[0] == '_' or s[0] in string.ascii_letters:
  for i in s:
   if i in string.ascii_letters or i == '_' or i in string.digits:
    pass
   else:
    return 0
  return 1
 else:
  return 0


def recognition(filename):
 try:
  fp_read = open(filename, 'r')
  string = ""
  sign = 0
  while True:
   read = fp_read.read(1)
   if not read:
    break

   if read == ' ':
    if len(string.strip()) < 1:
     sign = 0
     pass
    else:
     if sign == 1 or sign == 2:
      string += read
     else:
      save(string)
      string = ""
      sign = 0
   elif read == '(':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('(')
   elif read == ')':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(')')
   elif read == '[':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('[')
   elif read == ']':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(']')
   elif read == '{':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('{')
   elif read == '}':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('}')
   elif read == '<':
    save(string)
    string = ""
    save('<')
   elif read == '>':
    save(string)
    string = ""
    save('>')
   elif read == ',':
    save(string)
    string = ""
    save(',')
   elif read == "'":
    string += read
    if sign == 1:
     sign = 0
     save_const(string)
     string = ""
    else:
     if sign != 2:
      sign = 1
   elif read == '"':
    string += read
    if sign == 2:
     sign = 0
     save_const(string)
     string = ""
    else:
     if sign != 1:
      sign = 2
   elif read == ':':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(':')
   elif read == '+':
    save(string)
    string = ""
    save('+')
   elif read == '=':
    save(string)
    string = ""
    save('=')
   else:
    string += read

 except Exception as e:
  print(e)


def main():
 if len(sys.argv) < 2:
  print("Please Input FileName")
 else:
  pretreatment(sys.argv[1])
 recognition('file.tmp')
 for i in signlist.keys():
  print("(", signlist[i], ",", i, ")")


if __name__ == '__main__':
 main()

以上就是本文的全部内容,希望对大家的学习有所帮助,也希望大家多多支持三水点靠木。

Python 相关文章推荐
Python实现的金山快盘的签到程序
Jan 17 Python
Python随机生成一个6位的验证码代码分享
Mar 24 Python
python中利用Future对象回调别的函数示例代码
Sep 07 Python
Python编程求解二叉树中和为某一值的路径代码示例
Jan 04 Python
200 行python 代码实现 2048 游戏
Jan 12 Python
Python基于SMTP协议实现发送邮件功能详解
Aug 14 Python
Python小工具之消耗系统指定大小内存的方法
Dec 03 Python
pandas dataframe的合并实现(append, merge, concat)
Jun 24 Python
python自动化工具之pywinauto实例详解
Aug 26 Python
python Dijkstra算法实现最短路径问题的方法
Sep 19 Python
Python字典深浅拷贝与循环方式方法详解
Feb 09 Python
在Mac中PyCharm配置python Anaconda环境过程图解
Mar 11 Python
使用Python 统计高频字数的方法
Jan 31 #Python
python3正则提取字符串里的中文实例
Jan 31 #Python
python统计中文字符数量的两种方法
Jan 31 #Python
解决python写入带有中文的字符到文件错误的问题
Jan 31 #Python
在Python中居然可以定义两个同名通参数的函数
Jan 31 #Python
Python的条件表达式和lambda表达式实例
Jan 31 #Python
OpenCV+Python识别车牌和字符分割的实现
Jan 31 #Python
You might like
php 数组二分法查找函数代码
2010/02/16 PHP
Yii框架调试心得--在页面输出执行sql语句
2014/12/25 PHP
Yii 2.0如何使用页面缓存方法示例
2017/05/23 PHP
PHP基于DateTime类解决Unix时间戳与日期互转问题【针对1970年前及2038年后时间戳】
2018/06/13 PHP
ExtJS4 Grid改变单元格背景颜色及Column render学习
2013/02/06 Javascript
js使浏览器窗口最大化实现代码(适用于IE)
2013/08/07 Javascript
JavaScript按位运算符的应用简析
2014/02/04 Javascript
Js和JQuery获取鼠标指针坐标的实现代码分享
2015/05/25 Javascript
jquery京东商城双11焦点图多图广告特效代码分享
2015/09/06 Javascript
基于Echarts 3.19 制作常用的图形(非静态)
2016/05/19 Javascript
利用js来实现缩略语列表、文献来源链接和快捷键列表
2016/12/16 Javascript
jQuery实现移动端Tab选项卡效果
2017/03/15 Javascript
Vue路由跳转问题记录详解
2017/06/15 Javascript
js CSS3实现卡牌旋转切换效果
2017/07/04 Javascript
详解Nodejs 通过 fs.createWriteStream 保存文件
2017/10/10 NodeJs
Vue自定义属性实例分析
2019/02/23 Javascript
python 查找文件夹下所有文件 实现代码
2009/07/01 Python
python+matplotlib绘制简单的海豚(顶点和节点的操作)
2018/01/02 Python
解决csv.writer写入文件有多余的空行问题
2018/07/06 Python
详解Python的三种拷贝方式
2020/02/11 Python
浅谈tensorflow模型保存为pb的各种姿势
2020/05/25 Python
python3中celery异步框架简单使用+守护进程方式启动
2021/01/20 Python
使用HTML5里的classList操作CSS类
2016/06/28 HTML / CSS
移动端Html5页面生成图片解决方案
2018/08/07 HTML / CSS
日本运动品牌美津浓官方购物网站:MIZUNO SHOP
2016/08/21 全球购物
斯凯奇新西兰官网:SKECHERS新西兰
2018/02/22 全球购物
Why do we need Unit test
2013/01/03 面试题
大学生学习生活的自我评价
2013/11/01 职场文书
高中班长自我鉴定
2013/12/20 职场文书
毕业生个人自荐书
2015/03/05 职场文书
2015年小学教科研工作总结
2015/07/20 职场文书
2016党员入党决心书
2015/09/22 职场文书
《有余数的除法》教学反思
2016/02/22 职场文书
Python中Cookies导出某站用户数据的方法
2021/05/17 Python
python调用ffmpeg命令行工具便捷操作视频示例实现过程
2021/11/01 Python
Nginx使用ngx_http_upstream_module实现负载均衡功能示例
2022/08/05 Servers