python实现词法分析器


Posted in Python onJanuary 31, 2019

简单Python词法分析器实现,供大家参考,具体内容如下

词法分析器状态转换图:

python实现词法分析器

词法分析器总流程图:

python实现词法分析器

预处理程序:

python实现词法分析器

词法分析器:

python实现词法分析器

词法分析器程序详细设计

python实现词法分析器

详细代码实现:

#!/usr/bin/env python3.4
# coding=utf-8
import sys
import string


keywards = {}

# 关键字部分
keywards['False'] = 101
keywards['class'] = 102
keywards['finally'] = 103
keywards['is'] = 104
keywards['return'] = 105
keywards['None'] = 106
keywards['continue'] = 107
keywards['for'] = 108
keywards['lambda'] = 109
keywards['try'] = 110
keywards['True'] = 111
keywards['def'] = 112
keywards['from'] = 113
keywards['nonlocal'] = 114
keywards['while'] = 115
keywards['and'] = 116
keywards['del'] = 117
keywards['global'] = 118
keywards['not'] = 119
keywards['with'] = 120
keywards['as'] = 121
keywards['elif'] = 122
keywards['if'] = 123
keywards['or'] = 124
keywards['yield'] = 125
keywards['assert'] = 126
keywards['else'] = 127
keywards['import'] = 128
keywards['pass'] = 129
keywards['break'] = 130
keywards['except'] = 131
keywards['in'] = 132
keywards['raise'] = 133

# 符号
keywards['+'] = 201
keywards['-'] = 202
keywards['*'] = 203
keywards['/'] = 204
keywards['='] = 205
keywards[':'] = 206
keywards['<'] = 207
keywards['>'] = 208
keywards['%'] = 209
keywards['&'] = 210
keywards['!'] = 211
keywards['('] = 212
keywards[')'] = 213
keywards['['] = 214
keywards[']'] = 215
keywards['{'] = 216
keywards['}'] = 217
keywards['#'] = 218
keywards['|'] = 219
keywards[','] = 220
# 变量
# keywards['var'] = 301

# 常量
# keywards['const'] = 401

# Error
# keywards['const'] = 501

signlist = {}


# 预处理函数,将文件中的空格,换行等无关字符处理掉
def pretreatment(file_name):
 try:
  fp_read = open(file_name, 'r')
  fp_write = open('file.tmp', 'w')
  sign = 0
  while True:
   read = fp_read.readline()
   if not read:
    break
   length = len(read)
   i = -1
   while i < length - 1:
    i += 1
    if sign == 0:
     if read[i] == ' ':
      continue
    if read[i] == '#':
     break
    elif read[i] == ' ':
     if sign == 1:
      continue
     else:
      sign = 1
      fp_write.write(' ')
    elif read[i] == '\t':
     if sign == 1:
      continue
     else:
      sign = 1
      fp_write.write(' ')
    elif read[i] == '\n':
     if sign == 1:
      continue
     else:
      fp_write.write(' ')
      sign = 1
    elif read[i] == '"':
     fp_write.write(read[i])
     i += 1
     while i < length and read[i] != '"':
      fp_write.write(read[i])
      i += 1
     if i >= length:
      break
     fp_write.write(read[i])
    elif read[i] == "'":
     fp_write.write(read[i])
     i += 1
     while i < length and read[i] != "'":
      fp_write.write(read[i])
      i += 1
     if i >= length:
      break
     fp_write.write(read[i])
    else:
     sign = 3
     fp_write.write(read[i])
 except Exception:
  print(file_name, ': This FileName Not Found!')


def save(string):
 if string in keywards.keys():
  if string not in signlist.keys():
   signlist[string] = keywards[string]
 else:
  try:
   float(string)
   save_const(string)
  except ValueError:
   save_var(string)


def save_var(string):
 if string not in signlist.keys():
  if len(string.strip()) < 1:
   pass
  else:
   if is_signal(string) == 1:
    signlist[string] = 301
   else:
    signlist[string] = 501


def save_const(string):
 if string not in signlist.keys():
  signlist[string] = 401


def save_error(string):
 if string not in signlist.keys():
  signlist[string] = 501


def is_signal(s):
 if s[0] == '_' or s[0] in string.ascii_letters:
  for i in s:
   if i in string.ascii_letters or i == '_' or i in string.digits:
    pass
   else:
    return 0
  return 1
 else:
  return 0


def recognition(filename):
 try:
  fp_read = open(filename, 'r')
  string = ""
  sign = 0
  while True:
   read = fp_read.read(1)
   if not read:
    break

   if read == ' ':
    if len(string.strip()) < 1:
     sign = 0
     pass
    else:
     if sign == 1 or sign == 2:
      string += read
     else:
      save(string)
      string = ""
      sign = 0
   elif read == '(':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('(')
   elif read == ')':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(')')
   elif read == '[':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('[')
   elif read == ']':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(']')
   elif read == '{':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('{')
   elif read == '}':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('}')
   elif read == '<':
    save(string)
    string = ""
    save('<')
   elif read == '>':
    save(string)
    string = ""
    save('>')
   elif read == ',':
    save(string)
    string = ""
    save(',')
   elif read == "'":
    string += read
    if sign == 1:
     sign = 0
     save_const(string)
     string = ""
    else:
     if sign != 2:
      sign = 1
   elif read == '"':
    string += read
    if sign == 2:
     sign = 0
     save_const(string)
     string = ""
    else:
     if sign != 1:
      sign = 2
   elif read == ':':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(':')
   elif read == '+':
    save(string)
    string = ""
    save('+')
   elif read == '=':
    save(string)
    string = ""
    save('=')
   else:
    string += read

 except Exception as e:
  print(e)


def main():
 if len(sys.argv) < 2:
  print("Please Input FileName")
 else:
  pretreatment(sys.argv[1])
 recognition('file.tmp')
 for i in signlist.keys():
  print("(", signlist[i], ",", i, ")")


if __name__ == '__main__':
 main()

以上就是本文的全部内容,希望对大家的学习有所帮助,也希望大家多多支持三水点靠木。

Python 相关文章推荐
举例讲解Python程序与系统shell交互的方式
Apr 09 Python
pandas中的DataFrame按指定顺序输出所有列的方法
Apr 10 Python
Python Logging 日志记录入门学习
Jun 02 Python
Python读取mat文件,并保存为pickle格式的方法
Oct 23 Python
Django 开发环境配置过程详解
Jul 18 Python
python银行系统实现源码
Oct 25 Python
python游戏开发的五个案例分享
Mar 09 Python
python实现udp传输图片功能
Mar 20 Python
Python语法垃圾回收机制原理解析
Mar 25 Python
python初步实现word2vec操作
Jun 09 Python
python如何爬取动态网站
Sep 09 Python
详解用python -m http.server搭一个简易的本地局域网
Sep 24 Python
使用Python 统计高频字数的方法
Jan 31 #Python
python3正则提取字符串里的中文实例
Jan 31 #Python
python统计中文字符数量的两种方法
Jan 31 #Python
解决python写入带有中文的字符到文件错误的问题
Jan 31 #Python
在Python中居然可以定义两个同名通参数的函数
Jan 31 #Python
Python的条件表达式和lambda表达式实例
Jan 31 #Python
OpenCV+Python识别车牌和字符分割的实现
Jan 31 #Python
You might like
PHP垃圾回收机制简单说明
2010/07/22 PHP
PHP函数http_build_query使用详解
2014/08/20 PHP
Chrome Web App开发小结
2014/09/04 PHP
ThinkPHP框架中使用Memcached缓存数据的方法
2018/03/31 PHP
php微信公众号开发之快递查询
2018/10/20 PHP
javascript 子窗体父窗体相互传值方法
2010/05/31 Javascript
原生javaScript做得动态表格(注释写的很清楚)
2013/12/29 Javascript
基于jquery的文字向上跑动类似跑马灯的效果
2014/09/22 Javascript
JS日期格式化之javascript Date format
2015/10/01 Javascript
分享两段简单的JS代码防止SQL注入
2016/04/12 Javascript
JS封装的自动创建表格的实现代码
2016/06/15 Javascript
解决Node.js使用MySQL出现connect ECONNREFUSED 127.0.0.1:3306的问题
2017/03/09 Javascript
ExtJs使用自定义插件动态保存表头配置(隐藏或显示)
2018/09/25 Javascript
webpack+vue-cli项目中引入外部非模块格式js的方法
2018/09/28 Javascript
30分钟用Node.js构建一个API服务器的步骤详解
2019/05/24 Javascript
JS学习笔记之原型链和利用原型实现继承详解
2019/05/29 Javascript
简单了解Vue + ElementUI后台管理模板
2020/04/07 Javascript
基于JS实现视频上传显示进度条
2020/05/12 Javascript
Vue微信公众号网页分享的示例代码
2020/05/28 Javascript
详解JavaScript 异步编程
2020/07/13 Javascript
Python 错误和异常小结
2013/10/09 Python
Python SQLite3数据库操作类分享
2014/06/10 Python
详解Python3中yield生成器的用法
2015/08/20 Python
Python 实现链表实例代码
2017/04/07 Python
Python中异常重试的解决方案详解
2017/05/05 Python
Python实现字典的遍历与排序功能示例
2017/12/23 Python
python进行文件对比的方法
2018/12/24 Python
在python里协程使用同步锁Lock的实例
2019/02/19 Python
在jupyter notebook 添加 conda 环境的操作详解
2020/04/10 Python
全球领先的美容用品专卖店:Beauty Plus Salon
2018/09/04 全球购物
美国婴儿服装购物网站:Gerber Childrenswear
2020/05/06 全球购物
为什么要优先使用同步代码块而不是同步方法?
2013/01/30 面试题
EJB的角色和三个对象
2015/12/31 面试题
医院护士见习期自我鉴定
2014/04/10 职场文书
校外活动方案
2014/08/28 职场文书
Redis实现主从复制方式(Master&Slave)
2022/06/21 Redis