python处理csv数据的方法


Posted in Python onMarch 11, 2015

本文实例讲述了python处理csv数据的方法。分享给大家供大家参考。具体如下:

Python代码:

#coding=utf-8

__author__ = 'dehua.li'

from datetime import *

import datetime

import csv

import sys

import time

import string

import os

import os.path

import pylab as plt

rootdir='/nethome/dehua.li/orderlifeCycleData/xingzheng'

writeFileDir="/nethome/dehua.li/orderlifeMyWork/xingzheng/csv"

heyueFile="/nethome/dehua.li/orderlifeCycleData/heyue_150128.csv"

ms_acked="1"

msg=[]

ex=[]

def getTheDate(date):

    [filenamePart1,filenamePart2]=string.split(filename,'.')

    [filenamePart11,filenamePart12,filenamePart13]=string.split(filenamePart1,'_')

    return filenamePart13

LocalTime=datetime.datetime.fromtimestamp(time.mktime(time.strptime("2014-11-04 20:59:59","%Y-%m-%d %H:%M:%S")))

for parent,dirname,filenames in os.walk(rootdir):

    for filename in filenames:

        fileNameWrite=os.path.join(writeFileDir,filename)

        print fileNameWrite

        csvfile00=open(fileNameWrite,'wb')

        writer1=csv.writer(csvfile00)

        writer1.writerow(['FeedCode','OrderId','Status','LocalTime','Time','Exchange'])

        fileName=os.path.join(parent,filename)

        [filenamePart1,filenamePart2]=string.split(filename,'.')

        [filenamePart11,filenamePart12,filenamePart13]=string.split(filenamePart1,'_')

        #filenamePart11_filenamePart12_filenamePart13.filenamePart2:dongzheng_orderlifeCycleData_20150111.csv

        print fileName

        with open(fileName,'rb') as csvfile:

            reader=csv.reader(csvfile)

            CsvItem=[row for row in reader]

            for item in CsvItem:

                if item[3]=='TPO':

                    #print " filter TPO "

                    continue

                if item[12]=='Sent':

                    [tm_local,ms_local]=string.split(item[15],'.')

                    [tm_localup,ms_localup]=string.split(item[19],'.')

                    LocalTime=datetime.datetime.fromtimestamp(time.mktime(time.strptime(tm_local,"%Y-%m-%d %H:%M:%S")))

                    LocalUpdate=datetime.datetime.fromtimestamp(time.mktime(time.strptime(tm_localup,"%Y-%m-%d %H:%M:%S")))

                    tm=int(((LocalTime-LocalUpdate).seconds))*1000

                    ms_sent=str(int(ms_local)-int(ms_localup)+tm)

                    if int(ms_sent)>10*60*1000:

                        print "ms_sent>600000"

                        continue

                    if(int(ms_local)-int(ms_localup)+tm)<0:

                        print 'wrong1'

                    msg=[]

                    msg.append(item[0])

                    msg.append(item[1])

                    msg.append(item[12])

                    msg.append(item[15])

                    msg.append(ms_sent)

                    with open(heyueFile,'rb') as csvfile1:

                        reader=csv.reader(csvfile1)

                        CsvItem=[row for row in reader]

                        for Item in CsvItem:

                            if Item[1]==item[0]:

                                msg.append(Item[3])

                                writer1.writerow(msg)

                                #print 'write ok'

                                ex=Item[3]

                                break

                    csvfile1.close()

                    with open(fileName,'rb') as csvfile22:

                        reader=csv.reader(csvfile22)

                        CsvItem2=[row for row in reader]

                        for item_ in CsvItem2:

                            if item_[12]=='Acked' and item_[1]==item[1]:

                                [tm_local2,ms_local2]=string.split(item_[15],'.')

                                LocalTime2=datetime.datetime.fromtimestamp(time.mktime(time.strptime(tm_local2,"%Y-%m-%d %H:%M:%S")))

                                tm2=int(((LocalTime2-LocalTime).seconds))*1000

                                ms_acked=str(int(ms_local2)-int(ms_local)+tm2)

                                if int(ms_acked)>10*60*1000:

                                    print "MSacked>600000"

                                    continue

                                msg=[]

                                msg.append(item_[0])

                                msg.append(item_[1])

                                msg.append(item_[12])

                                msg.append(item_[15])

                                msg.append(ms_acked)

                    with open(heyueFile,'rb') as csvfile111:

                        reader=csv.reader(csvfile111)

                        CsvItem=[row for row in reader]

                        for Item in CsvItem:

                            if Item[1]==item[0]:

                                msg.append(Item[3])

                                writer1.writerow(msg)

                                #print 'write ok'

                                break

                                #print "write ok"

                    csvfile22.close()

        csvfile.close()

        csvfile00.close()
#coding=utf-8

#__author__ = 'dehua.li'

from datetime import *

import datetime

import csv

import sys

import time

import string

import os

import os.path

import pylab as plt

def median(lst):

    even = (0 if len(lst) % 2 else 1) + 1

    half = (len(lst) - 1) / 2

    return sum(sorted(lst)[half:half + even]) / float(even)

def mean(lst):

    if len(lst)==0:

        return 0

    return sum(lst)/len(lst)

nightLine="21:01:00"

morningLine="09:01:00"

def getTheDate(date):

    [filenamePart1,filenamePart2]=string.split(filename,'.')

    [filenamePart11,filenamePart12,filenamePart13]=string.split(filenamePart1,'_')

    return filenamePart13

def afterOneMin(time):

    [tm_local,ms_local]=string.split(time,'.')

    [ymd,hms]=string.split(tm_local,' ')

    flag=0

    if hms>"21:01:00":

        flag=1

    elif hms>"09:01:00" and hms<"20:00:00":

        flag=1

    elif hms>"00:00:00" and hms<"05:00:00":

        flag=1

    return flag

rootdir="/nethome/dehua.li/orderlifeMyWork/xingzheng/csv"

#csvfileMaxMin = open('e:\dehua.li\csv\__xingzhenMaxMin.csv','wb')

#writer1 = csv.writer(csvfileMaxMin)

#writer1.writerow(['FeedCode','date','SentMaxTime','SentMaxLocalTime','SentMinTime','SentMinLocalTime','SentMeanTime','SentMedian','AckedMaxTime','AckedMaxLocalTime','AckedMinTime','AckedMinLocalTime','AckedMeanTime','AckedMedianTime','Exchange'])

#writer1.writerow(['FeedCode','date','SentMaxTime','SentMinTime','SentMeanTime','SentMedian','AckedMaxTime','AckedMinTime','AckedMeanTime','AckedMedianTime','Exchange'])

msg=[]

codeList=list()

orderList=list()

itemSentList=[]

itemAckedList=[]

feedCode=[]

exchange=[]

zhengshangSentMedian=0

zhengshangSentMean=0

zhengshangAckedMedian=0

zhengshangAckedMean=0

dashangSentMedian=0

dashangSentMean=0

dashangAckedMedian=0

dashangAckedMean=0

shangqiSentMedian=0

shangqiSentMean=0

shangqiAckedMedian=0

shangqiAckedMean=0

zhongjinSentMedian=0

zhongjinSentMean=0

zhongjinAckedMedian=0

zhongjinAckedMean=0

zhengshangSent=[]

zhengshangAcked=[]

dashangSent=[]

dashangAcked=[]

shangqiSent=[]

shangqiAcked=[]

zhongjinSent=[]

zhongjinAcked=[]

zhengshangSentMedianAll=[]

zhengshangSentMeanAll=[]

zhengshangAckedMedianAll=[]

zhengshangAckedMeanAll=[]

dashangSentMedianAll=[]

dashangSentMeanAll=[]

dashangAckedMedianAll=[]

dashangAckedMeanAll=[]

shangqiSentMedianAll=[]

shangqiSentMeanAll=[]

shangqiAckedMedianAll=[]

shangqiAckedMeanAll=[]

zhongjinSentMedianAll=[]

zhongjinSentMeanAll=[]

zhongjinAckedMedianAll=[]

zhongjinAckedMeanAll=[]

zhengshang='0'

dashang='0'

shangqi='0'

zhongjin='0'

with open('/nethome/dehua.li/orderlifeCycleData/heyue_150128.csv','rb') as csvfile:

    reader=csv.reader(csvfile)

    csvItem=[row for row in reader]

    zhengshang=csvItem[300][3]

    dashang=csvItem[5][3]

    shangqi=csvItem[165][3]

    zhongjin=csvItem[435][3]

    #for item in csvItem:

    #    if item[3]==zhengshang:

     #       print item

for parent,dirname,filenames in os.walk(rootdir):

    for filename in filenames:

        fileName=os.path.join(rootdir,filename)

        csvfile1=open(fileName,'rb')

        reader=csv.reader(csvfile1)

        CsvItem=[row for row in reader]

        for item in CsvItem:

            if item[0]=='FeedCode':

                continue

            if item[0] not in codeList:

                codeList.append(item[0])

                #print CsvItem[15]

            if len(item)<=5:

                print fileName

                print item

                print '++++++++++++++++++++++++++++++'

            #if afterOneMin(item[3])==0:

            #    print item[3]

            #    continue

            if item[5]==zhengshang and item[2]=='Sent':

                zhengshangSent.append(int(item[4]))

            elif item[5]==zhengshang and item[2]=='Acked':

                zhengshangAcked.append(int(item[4]))

            elif item[5]==dashang and item[2]=='Sent':

                 dashangSent.append(int(item[4]))

            elif item[5]==dashang and item[2]=='Acked':

                 dashangAcked.append(int(item[4]))

            elif item[5]==shangqi and item[2]=='Sent':

                 shangqiSent.append(int(item[4]))

                 if int(item[4])>=600000:

                     print "------------"

                     print item

            elif item[5]==shangqi and item[2]=='Acked':

                 shangqiAcked.append(int(item[4]))

            elif item[5]==zhongjin and item[2]=='Sent':

                 zhongjinSent.append(int(item[4]))

            elif item[5]==zhongjin and item[2]=='Acked':

                 zhongjinAcked.append(int(item[4]))

            else:

                 print "wrong info"

                 print item

        if mean(shangqiSent)>420000:

            print sum(shangqiSent)

            print len(shangqiSent)

            print item

            print fileName

            print shangqiSent

        zhengshangSentMedian=median(zhengshangSent)

        zhengshangSentMean=mean(zhengshangSent)

        zhengshangAckedMedian=median(zhengshangAcked)

        zhengshangAckedMean=mean(zhengshangAcked)

        dashangSentMedian=median(dashangSent)

        dashangSentMean=mean(dashangSent)

        dashangAckedMedian=median(dashangAcked)

        dashangAckedMean=mean(dashangAcked)

        shangqiSentMedian=median(shangqiSent)

        shangqiSentMean=mean(shangqiSent)

        shangqiAckedMedian=median(shangqiAcked)

        shangqiAckedMean=mean(shangqiAcked)

        zhongjinSentMedian=median(zhongjinSent)

        zhongjinSentMean=mean(zhongjinSent)

        zhongjinAckedMedian=median(zhongjinAcked)

        zhongjinAckedMean=mean(zhongjinAcked)

        #if mean(shangqiSent)>70:

        #    print '================================'

        #    print fileName

        #print codeList

        '''

        for listItem in codeList:

            itemSentList=[]

            itemAckedList=[]

            for item in CsvItem:

                if item[0]==listItem and item[2]=='Sent':

                    itemSentList.append(int(item[4]))

                    exchange=item[5]

                elif  item[0]==listItem and item[2]=='Acked':

                    itemAckedList.append(int(item[4]))

            #print itemSentList

            itemMaxSent=max(itemSentList)

            itemMinSent=min(itemSentList)

            itemAvgSent=sum(itemSentList)/len(itemSentList)

            itemMaxAcked=max(itemAckedList)

            itemMinAcked=min(itemAckedList)

            itemAvgAcked=sum(itemAckedList)/len(itemAckedList)

            SentMedian=median(itemSentList)

            AckedMedian=median(itemAckedList)

            msg=[]

            msg.append(listItem)         #0

            msg.append("2015/01/14")   #1

            msg.append(itemMaxSent)     #2

            msg.append(itemMinSent)     #3

            msg.append(itemAvgSent)     #4

            msg.append(SentMedian)     #5

            msg.append(itemMaxAcked)    #6

            msg.append(itemMinAcked)     #7

            msg.append(itemAvgAcked)     #8

            msg.append(AckedMedian)      #9

            msg.append(exchange)          #10

            if len(msg)>15:

               print "------------------------------"

               print msg

            writer1.writerow(msg)

         '''

        zhengshangSentMedianAll.append(zhengshangSentMedian)

        zhengshangSentMeanAll.append(zhengshangSentMean)

        zhengshangAckedMedianAll.append(zhengshangAckedMedian)

        zhengshangAckedMeanAll.append(zhengshangAckedMean)

        dashangSentMedianAll.append(dashangSentMedian)

        dashangSentMeanAll.append(dashangSentMean)

        dashangAckedMedianAll.append(dashangAckedMedian)

        dashangAckedMeanAll.append(dashangAckedMean)

        shangqiSentMedianAll.append(shangqiSentMedian)

        shangqiSentMeanAll.append(shangqiSentMean)

        shangqiAckedMedianAll.append(shangqiAckedMedian)

        shangqiAckedMeanAll.append(shangqiAckedMean)

        zhongjinSentMedianAll.append(zhongjinSentMedian)

        zhongjinSentMeanAll.append(zhongjinSentMean)

        zhongjinAckedMedianAll.append(zhongjinAckedMedian)

        zhongjinAckedMeanAll.append(zhongjinAckedMean)

plt.figure(1)

plt.figure(2)

plt.figure(3)

plt.figure(4)

plt.figure(1)

plt.title('SentMean r-zhengshang b-dashang,green-shangqi grey-zhongjin')

plt.plot(range(1,len(zhengshangSentMeanAll)+1),zhengshangSentMeanAll,'r')

plt.plot(range(1,len(dashangSentMeanAll)+1),dashangSentMeanAll,'b')

plt.plot(range(1,len(shangqiSentMeanAll)+1),shangqiSentMeanAll,'g')

plt.plot(range(1,len(zhongjinSentMeanAll)+1),zhongjinSentMeanAll,'grey')

plt.savefig('/nethome/dehua.li/orderlifeMyWork/xingzheng/data_noTPO_in10minutes/SentMean.png')

plt.figure(2)

plt.title('SentMedian r-zhengshang b-dashang,green-shangqi grey-zhongjin')

plt.plot(range(1,len(zhengshangSentMedianAll)+1),zhengshangSentMedianAll,'r')

plt.plot(range(1,len(dashangSentMedianAll)+1),dashangSentMedianAll,'b')

plt.plot(range(1,len(shangqiSentMedianAll)+1),shangqiSentMedianAll,'g')

plt.plot(range(1,len(zhongjinSentMedianAll)+1),zhongjinSentMedianAll,'grey')

plt.savefig('/nethome/dehua.li/orderlifeMyWork/xingzheng/data_noTPO_in10minutes/SentMedian.png')

plt.figure(3)

plt.title('AckedMean r-zhengshang b-dashang,green-shangqi grey-zhongjin')

plt.plot(range(1,len(zhengshangAckedMeanAll)+1),zhengshangAckedMeanAll,'r')

plt.plot(range(1,len(dashangAckedMeanAll)+1),dashangAckedMeanAll,'b')

plt.plot(range(1,len(shangqiAckedMeanAll)+1),shangqiAckedMeanAll,'g')

plt.plot(range(1,len(zhongjinAckedMeanAll)+1),zhongjinAckedMeanAll,'grey')

plt.savefig('/nethome/dehua.li/orderlifeMyWork/xingzheng/data_noTPO_in10minutes/AckedMean.png')

plt.figure(4)

plt.title('AckedMedian r-zhengshang b-dashang,green-shangqi grey-zhongjin')

plt.plot(range(1,len(zhengshangAckedMedianAll)+1),zhengshangAckedMedianAll,'r')

plt.plot(range(1,len(dashangAckedMedianAll)+1),dashangAckedMedianAll,'b')

plt.plot(range(1,len(shangqiAckedMedianAll)+1),shangqiAckedMedianAll,'g')

plt.plot(range(1,len(zhongjinAckedMedianAll)+1),zhongjinAckedMedianAll,'grey')

plt.savefig('/nethome/dehua.li/orderlifeMyWork/xingzheng/data_noTPO_in10minutes/AckedMedian.png')

plt.show()

print 'over'

希望本文所述对大家的Python程序设计有所帮助。

Python 相关文章推荐
python调用Moxa PCOMM Lite通过串口Ymodem协议实现发送文件
Aug 15 Python
浅析Python的web.py框架中url的设定方法
Jul 11 Python
详解python 字符串和日期之间转换 StringAndDate
May 04 Python
django中send_mail功能实现详解
Feb 06 Python
Python快速查找list中相同部分的方法
Jun 27 Python
Django压缩静态文件的实现方法详析
Aug 26 Python
python实现多进程代码示例
Oct 31 Python
Python最小二乘法矩阵
Jan 02 Python
安装Pycharm2019以及配置anconda教程的方法步骤
Nov 11 Python
python的slice notation的特殊用法详解
Dec 27 Python
Windows环境下Python3.6.8 importError: DLLload failed:找不到指定的模块
Nov 01 Python
Python selenium模拟网页点击爬虫交管12123违章数据
May 26 Python
python模拟鼠标拖动操作的方法
Mar 11 #Python
Python创建系统目录的方法
Mar 11 #Python
Python实现从订阅源下载图片的方法
Mar 11 #Python
Python获取网页上图片下载地址的方法
Mar 11 #Python
Python实现从百度API获取天气的方法
Mar 11 #Python
Python计算回文数的方法
Mar 11 #Python
Python用GET方法上传文件
Mar 10 #Python
You might like
PHP+javascript模拟Matrix画面
2006/10/09 PHP
实用函数8
2007/11/08 PHP
thinkPHP中create方法与令牌验证实例浅析
2015/12/08 PHP
ThinkPHP3.2.3实现分页的方法详解
2016/06/03 PHP
PHP排序算法之归并排序(Merging Sort)实例详解
2018/04/21 PHP
PHP7 新增常量
2021/03/09 PHP
javascript 密码强度验证规则、打分、验证(给出前端代码,后端代码可根据强度规则翻译)
2010/05/18 Javascript
jQuery.extend()的实现方式详解及实例
2013/06/29 Javascript
javascript类型转换使用方法
2014/02/08 Javascript
jquery删除数据记录时的弹出提示效果
2014/05/06 Javascript
原生js实现复制对象、扩展对象 类似jquery中的extend()方法
2014/08/30 Javascript
javascript和jquery实现设置和移除文本框默认值效果代码
2015/01/13 Javascript
javascript实现控制文字大中小显示
2015/04/28 Javascript
JQuery操作textarea,input,select,checkbox方法
2015/09/02 Javascript
JavaScript代码实现禁止右键、禁选择、禁粘贴、禁shift、禁ctrl、禁alt
2015/11/17 Javascript
BootStrap Table对前台页面表格的支持实例讲解
2016/12/22 Javascript
jquery平滑滚动到顶部插件使用详解
2017/05/08 jQuery
jQuery图片缩放插件smartZoom使用实例详解
2017/08/25 jQuery
Vue上传组件vue Simple Uploader的用法示例
2017/08/25 Javascript
vue源码解析之事件机制原理
2018/04/21 Javascript
详解webpack模块加载器兼打包工具
2018/09/11 Javascript
小程序跳转到的H5页面再跳转回跳小程序的方法
2020/03/06 Javascript
jquery实现简单自动轮播图效果
2020/07/29 jQuery
[58:11]守擂赛第二周擂主赛 DeMonsTer vs Leopard
2020/04/28 DOTA
Python 绘图和可视化详细介绍
2017/02/11 Python
Python数据结构之哈夫曼树定义与使用方法示例
2018/04/22 Python
python+mysql实现教务管理系统
2019/02/20 Python
使用python将mysql数据库的数据转换为json数据的方法
2019/07/01 Python
python+OpenCV实现车牌号码识别
2019/11/08 Python
HTML5 Canvas——用路径描画线条实例介绍
2013/06/09 HTML / CSS
Ibood荷兰:互联网每日最佳在线优惠
2019/02/28 全球购物
小学生考试获奖感言
2014/01/30 职场文书
ktv中秋节活动方案
2014/01/30 职场文书
我的梦中国梦演讲稿
2014/04/23 职场文书
Ajax实现异步加载数据
2021/11/17 Javascript
解析python中的jsonpath 提取器
2022/01/18 Python