python实现词法分析器


Posted in Python onJanuary 31, 2019

简单Python词法分析器实现,供大家参考,具体内容如下

词法分析器状态转换图:

python实现词法分析器

词法分析器总流程图:

python实现词法分析器

预处理程序:

python实现词法分析器

词法分析器:

python实现词法分析器

词法分析器程序详细设计

python实现词法分析器

详细代码实现:

#!/usr/bin/env python3.4
# coding=utf-8
import sys
import string


keywards = {}

# 关键字部分
keywards['False'] = 101
keywards['class'] = 102
keywards['finally'] = 103
keywards['is'] = 104
keywards['return'] = 105
keywards['None'] = 106
keywards['continue'] = 107
keywards['for'] = 108
keywards['lambda'] = 109
keywards['try'] = 110
keywards['True'] = 111
keywards['def'] = 112
keywards['from'] = 113
keywards['nonlocal'] = 114
keywards['while'] = 115
keywards['and'] = 116
keywards['del'] = 117
keywards['global'] = 118
keywards['not'] = 119
keywards['with'] = 120
keywards['as'] = 121
keywards['elif'] = 122
keywards['if'] = 123
keywards['or'] = 124
keywards['yield'] = 125
keywards['assert'] = 126
keywards['else'] = 127
keywards['import'] = 128
keywards['pass'] = 129
keywards['break'] = 130
keywards['except'] = 131
keywards['in'] = 132
keywards['raise'] = 133

# 符号
keywards['+'] = 201
keywards['-'] = 202
keywards['*'] = 203
keywards['/'] = 204
keywards['='] = 205
keywards[':'] = 206
keywards['<'] = 207
keywards['>'] = 208
keywards['%'] = 209
keywards['&'] = 210
keywards['!'] = 211
keywards['('] = 212
keywards[')'] = 213
keywards['['] = 214
keywards[']'] = 215
keywards['{'] = 216
keywards['}'] = 217
keywards['#'] = 218
keywards['|'] = 219
keywards[','] = 220
# 变量
# keywards['var'] = 301

# 常量
# keywards['const'] = 401

# Error
# keywards['const'] = 501

signlist = {}


# 预处理函数,将文件中的空格,换行等无关字符处理掉
def pretreatment(file_name):
 try:
  fp_read = open(file_name, 'r')
  fp_write = open('file.tmp', 'w')
  sign = 0
  while True:
   read = fp_read.readline()
   if not read:
    break
   length = len(read)
   i = -1
   while i < length - 1:
    i += 1
    if sign == 0:
     if read[i] == ' ':
      continue
    if read[i] == '#':
     break
    elif read[i] == ' ':
     if sign == 1:
      continue
     else:
      sign = 1
      fp_write.write(' ')
    elif read[i] == '\t':
     if sign == 1:
      continue
     else:
      sign = 1
      fp_write.write(' ')
    elif read[i] == '\n':
     if sign == 1:
      continue
     else:
      fp_write.write(' ')
      sign = 1
    elif read[i] == '"':
     fp_write.write(read[i])
     i += 1
     while i < length and read[i] != '"':
      fp_write.write(read[i])
      i += 1
     if i >= length:
      break
     fp_write.write(read[i])
    elif read[i] == "'":
     fp_write.write(read[i])
     i += 1
     while i < length and read[i] != "'":
      fp_write.write(read[i])
      i += 1
     if i >= length:
      break
     fp_write.write(read[i])
    else:
     sign = 3
     fp_write.write(read[i])
 except Exception:
  print(file_name, ': This FileName Not Found!')


def save(string):
 if string in keywards.keys():
  if string not in signlist.keys():
   signlist[string] = keywards[string]
 else:
  try:
   float(string)
   save_const(string)
  except ValueError:
   save_var(string)


def save_var(string):
 if string not in signlist.keys():
  if len(string.strip()) < 1:
   pass
  else:
   if is_signal(string) == 1:
    signlist[string] = 301
   else:
    signlist[string] = 501


def save_const(string):
 if string not in signlist.keys():
  signlist[string] = 401


def save_error(string):
 if string not in signlist.keys():
  signlist[string] = 501


def is_signal(s):
 if s[0] == '_' or s[0] in string.ascii_letters:
  for i in s:
   if i in string.ascii_letters or i == '_' or i in string.digits:
    pass
   else:
    return 0
  return 1
 else:
  return 0


def recognition(filename):
 try:
  fp_read = open(filename, 'r')
  string = ""
  sign = 0
  while True:
   read = fp_read.read(1)
   if not read:
    break

   if read == ' ':
    if len(string.strip()) < 1:
     sign = 0
     pass
    else:
     if sign == 1 or sign == 2:
      string += read
     else:
      save(string)
      string = ""
      sign = 0
   elif read == '(':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('(')
   elif read == ')':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(')')
   elif read == '[':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('[')
   elif read == ']':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(']')
   elif read == '{':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('{')
   elif read == '}':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('}')
   elif read == '<':
    save(string)
    string = ""
    save('<')
   elif read == '>':
    save(string)
    string = ""
    save('>')
   elif read == ',':
    save(string)
    string = ""
    save(',')
   elif read == "'":
    string += read
    if sign == 1:
     sign = 0
     save_const(string)
     string = ""
    else:
     if sign != 2:
      sign = 1
   elif read == '"':
    string += read
    if sign == 2:
     sign = 0
     save_const(string)
     string = ""
    else:
     if sign != 1:
      sign = 2
   elif read == ':':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(':')
   elif read == '+':
    save(string)
    string = ""
    save('+')
   elif read == '=':
    save(string)
    string = ""
    save('=')
   else:
    string += read

 except Exception as e:
  print(e)


def main():
 if len(sys.argv) < 2:
  print("Please Input FileName")
 else:
  pretreatment(sys.argv[1])
 recognition('file.tmp')
 for i in signlist.keys():
  print("(", signlist[i], ",", i, ")")


if __name__ == '__main__':
 main()

以上就是本文的全部内容,希望对大家的学习有所帮助,也希望大家多多支持三水点靠木。

Python 相关文章推荐
Python模块学习 re 正则表达式
May 19 Python
python判断windows隐藏文件的方法
Mar 21 Python
Python入门篇之文件
Oct 20 Python
Python实现一个简单的MySQL类
Jan 07 Python
python静态方法实例
Jan 14 Python
利用python为运维人员写一个监控脚本
Mar 25 Python
对numpy中的where方法嵌套使用详解
Oct 31 Python
Pyinstaller打包.py生成.exe的方法和报错总结
Apr 02 Python
python GUI库图形界面开发之PyQt5线程类QThread详细使用方法
Feb 26 Python
详解PyQt5中textBrowser显示print语句输出的简单方法
Aug 07 Python
python 利用百度API识别图片文字(多线程版)
Dec 14 Python
matplotlib之pyplot模块坐标轴标签设置使用(xlabel()、ylabel())
Feb 22 Python
使用Python 统计高频字数的方法
Jan 31 #Python
python3正则提取字符串里的中文实例
Jan 31 #Python
python统计中文字符数量的两种方法
Jan 31 #Python
解决python写入带有中文的字符到文件错误的问题
Jan 31 #Python
在Python中居然可以定义两个同名通参数的函数
Jan 31 #Python
Python的条件表达式和lambda表达式实例
Jan 31 #Python
OpenCV+Python识别车牌和字符分割的实现
Jan 31 #Python
You might like
php图片加水印原理(超简单的实例代码)
2013/01/18 PHP
Linux php 中文乱码的快速解决方法
2016/05/13 PHP
PHP小程序支付功能完整版【基于thinkPHP】
2019/03/26 PHP
显示js对象所有属性和方法的函数
2009/10/16 Javascript
7个Javascript地图脚本整理
2009/10/20 Javascript
JS求平均值的小例子
2013/11/29 Javascript
jquery实现在页面加载的时自动为日期插件添加当前日期
2014/08/20 Javascript
Jquery实现图片预加载与延时加载的方法
2014/12/22 Javascript
分享使用AngularJS创建应用的5个框架
2015/12/05 Javascript
angularjs表格分页功能详解
2016/01/21 Javascript
js验证框架实现代码分享
2016/05/18 Javascript
浅谈React 属性和状态的一些总结
2016/11/21 Javascript
JS 终止执行的实现方法
2016/11/24 Javascript
Vue键盘事件用法总结
2017/04/18 Javascript
JS 组件系列之Bootstrap Table 冻结列功能IE浏览器兼容性问题解决方案
2017/06/30 Javascript
Bootstrap Table 删除和批量删除
2017/09/22 Javascript
详解extract-text-webpack-plugin 的使用及安装
2018/06/12 Javascript
CKEditor扩展插件:自动排版功能autoformat插件实现方法详解
2020/02/06 Javascript
JS pushlet XMLAdapter适配器用法案例解析
2020/10/16 Javascript
[01:08:48]LGD vs OG 2018国际邀请赛淘汰赛BO3 第三场 8.25
2018/08/29 DOTA
Python单体模式的几种常见实现方法详解
2017/07/28 Python
Python将图片转换为字符画的方法
2020/06/16 Python
PyTorch上搭建简单神经网络实现回归和分类的示例
2018/04/28 Python
Pycharm 如何一键加引号的方法步骤
2021/02/05 Python
Python 爬取淘宝商品信息栏目的实现
2021/02/06 Python
Python3压缩和解压缩实现代码
2021/03/01 Python
排序都有哪几种方法?请列举。用JAVA实现一个快速排序
2014/02/16 面试题
蜜蜂引路教学反思
2014/02/04 职场文书
毕业生简历自我评价范文
2014/04/09 职场文书
爱国主义教育演讲稿
2014/08/26 职场文书
卖车协议书范例
2014/09/16 职场文书
银行贷款收入证明
2014/10/17 职场文书
先进工作者申报材料
2014/12/23 职场文书
2019旅游导游工作总结
2019/06/27 职场文书
Nginx配置https原理及实现过程详解
2021/03/31 Servers
JS前端可视化canvas动画原理及其推导实现
2022/08/05 Javascript