python处理csv数据的方法


Posted in Python onMarch 11, 2015

本文实例讲述了python处理csv数据的方法。分享给大家供大家参考。具体如下:

Python代码:

#coding=utf-8

__author__ = 'dehua.li'

from datetime import *

import datetime

import csv

import sys

import time

import string

import os

import os.path

import pylab as plt

rootdir='/nethome/dehua.li/orderlifeCycleData/xingzheng'

writeFileDir="/nethome/dehua.li/orderlifeMyWork/xingzheng/csv"

heyueFile="/nethome/dehua.li/orderlifeCycleData/heyue_150128.csv"

ms_acked="1"

msg=[]

ex=[]

def getTheDate(date):

    [filenamePart1,filenamePart2]=string.split(filename,'.')

    [filenamePart11,filenamePart12,filenamePart13]=string.split(filenamePart1,'_')

    return filenamePart13

LocalTime=datetime.datetime.fromtimestamp(time.mktime(time.strptime("2014-11-04 20:59:59","%Y-%m-%d %H:%M:%S")))

for parent,dirname,filenames in os.walk(rootdir):

    for filename in filenames:

        fileNameWrite=os.path.join(writeFileDir,filename)

        print fileNameWrite

        csvfile00=open(fileNameWrite,'wb')

        writer1=csv.writer(csvfile00)

        writer1.writerow(['FeedCode','OrderId','Status','LocalTime','Time','Exchange'])

        fileName=os.path.join(parent,filename)

        [filenamePart1,filenamePart2]=string.split(filename,'.')

        [filenamePart11,filenamePart12,filenamePart13]=string.split(filenamePart1,'_')

        #filenamePart11_filenamePart12_filenamePart13.filenamePart2:dongzheng_orderlifeCycleData_20150111.csv

        print fileName

        with open(fileName,'rb') as csvfile:

            reader=csv.reader(csvfile)

            CsvItem=[row for row in reader]

            for item in CsvItem:

                if item[3]=='TPO':

                    #print " filter TPO "

                    continue

                if item[12]=='Sent':

                    [tm_local,ms_local]=string.split(item[15],'.')

                    [tm_localup,ms_localup]=string.split(item[19],'.')

                    LocalTime=datetime.datetime.fromtimestamp(time.mktime(time.strptime(tm_local,"%Y-%m-%d %H:%M:%S")))

                    LocalUpdate=datetime.datetime.fromtimestamp(time.mktime(time.strptime(tm_localup,"%Y-%m-%d %H:%M:%S")))

                    tm=int(((LocalTime-LocalUpdate).seconds))*1000

                    ms_sent=str(int(ms_local)-int(ms_localup)+tm)

                    if int(ms_sent)>10*60*1000:

                        print "ms_sent>600000"

                        continue

                    if(int(ms_local)-int(ms_localup)+tm)<0:

                        print 'wrong1'

                    msg=[]

                    msg.append(item[0])

                    msg.append(item[1])

                    msg.append(item[12])

                    msg.append(item[15])

                    msg.append(ms_sent)

                    with open(heyueFile,'rb') as csvfile1:

                        reader=csv.reader(csvfile1)

                        CsvItem=[row for row in reader]

                        for Item in CsvItem:

                            if Item[1]==item[0]:

                                msg.append(Item[3])

                                writer1.writerow(msg)

                                #print 'write ok'

                                ex=Item[3]

                                break

                    csvfile1.close()

                    with open(fileName,'rb') as csvfile22:

                        reader=csv.reader(csvfile22)

                        CsvItem2=[row for row in reader]

                        for item_ in CsvItem2:

                            if item_[12]=='Acked' and item_[1]==item[1]:

                                [tm_local2,ms_local2]=string.split(item_[15],'.')

                                LocalTime2=datetime.datetime.fromtimestamp(time.mktime(time.strptime(tm_local2,"%Y-%m-%d %H:%M:%S")))

                                tm2=int(((LocalTime2-LocalTime).seconds))*1000

                                ms_acked=str(int(ms_local2)-int(ms_local)+tm2)

                                if int(ms_acked)>10*60*1000:

                                    print "MSacked>600000"

                                    continue

                                msg=[]

                                msg.append(item_[0])

                                msg.append(item_[1])

                                msg.append(item_[12])

                                msg.append(item_[15])

                                msg.append(ms_acked)

                    with open(heyueFile,'rb') as csvfile111:

                        reader=csv.reader(csvfile111)

                        CsvItem=[row for row in reader]

                        for Item in CsvItem:

                            if Item[1]==item[0]:

                                msg.append(Item[3])

                                writer1.writerow(msg)

                                #print 'write ok'

                                break

                                #print "write ok"

                    csvfile22.close()

        csvfile.close()

        csvfile00.close()
#coding=utf-8

#__author__ = 'dehua.li'

from datetime import *

import datetime

import csv

import sys

import time

import string

import os

import os.path

import pylab as plt

def median(lst):

    even = (0 if len(lst) % 2 else 1) + 1

    half = (len(lst) - 1) / 2

    return sum(sorted(lst)[half:half + even]) / float(even)

def mean(lst):

    if len(lst)==0:

        return 0

    return sum(lst)/len(lst)

nightLine="21:01:00"

morningLine="09:01:00"

def getTheDate(date):

    [filenamePart1,filenamePart2]=string.split(filename,'.')

    [filenamePart11,filenamePart12,filenamePart13]=string.split(filenamePart1,'_')

    return filenamePart13

def afterOneMin(time):

    [tm_local,ms_local]=string.split(time,'.')

    [ymd,hms]=string.split(tm_local,' ')

    flag=0

    if hms>"21:01:00":

        flag=1

    elif hms>"09:01:00" and hms<"20:00:00":

        flag=1

    elif hms>"00:00:00" and hms<"05:00:00":

        flag=1

    return flag

rootdir="/nethome/dehua.li/orderlifeMyWork/xingzheng/csv"

#csvfileMaxMin = open('e:\dehua.li\csv\__xingzhenMaxMin.csv','wb')

#writer1 = csv.writer(csvfileMaxMin)

#writer1.writerow(['FeedCode','date','SentMaxTime','SentMaxLocalTime','SentMinTime','SentMinLocalTime','SentMeanTime','SentMedian','AckedMaxTime','AckedMaxLocalTime','AckedMinTime','AckedMinLocalTime','AckedMeanTime','AckedMedianTime','Exchange'])

#writer1.writerow(['FeedCode','date','SentMaxTime','SentMinTime','SentMeanTime','SentMedian','AckedMaxTime','AckedMinTime','AckedMeanTime','AckedMedianTime','Exchange'])

msg=[]

codeList=list()

orderList=list()

itemSentList=[]

itemAckedList=[]

feedCode=[]

exchange=[]

zhengshangSentMedian=0

zhengshangSentMean=0

zhengshangAckedMedian=0

zhengshangAckedMean=0

dashangSentMedian=0

dashangSentMean=0

dashangAckedMedian=0

dashangAckedMean=0

shangqiSentMedian=0

shangqiSentMean=0

shangqiAckedMedian=0

shangqiAckedMean=0

zhongjinSentMedian=0

zhongjinSentMean=0

zhongjinAckedMedian=0

zhongjinAckedMean=0

zhengshangSent=[]

zhengshangAcked=[]

dashangSent=[]

dashangAcked=[]

shangqiSent=[]

shangqiAcked=[]

zhongjinSent=[]

zhongjinAcked=[]

zhengshangSentMedianAll=[]

zhengshangSentMeanAll=[]

zhengshangAckedMedianAll=[]

zhengshangAckedMeanAll=[]

dashangSentMedianAll=[]

dashangSentMeanAll=[]

dashangAckedMedianAll=[]

dashangAckedMeanAll=[]

shangqiSentMedianAll=[]

shangqiSentMeanAll=[]

shangqiAckedMedianAll=[]

shangqiAckedMeanAll=[]

zhongjinSentMedianAll=[]

zhongjinSentMeanAll=[]

zhongjinAckedMedianAll=[]

zhongjinAckedMeanAll=[]

zhengshang='0'

dashang='0'

shangqi='0'

zhongjin='0'

with open('/nethome/dehua.li/orderlifeCycleData/heyue_150128.csv','rb') as csvfile:

    reader=csv.reader(csvfile)

    csvItem=[row for row in reader]

    zhengshang=csvItem[300][3]

    dashang=csvItem[5][3]

    shangqi=csvItem[165][3]

    zhongjin=csvItem[435][3]

    #for item in csvItem:

    #    if item[3]==zhengshang:

     #       print item

for parent,dirname,filenames in os.walk(rootdir):

    for filename in filenames:

        fileName=os.path.join(rootdir,filename)

        csvfile1=open(fileName,'rb')

        reader=csv.reader(csvfile1)

        CsvItem=[row for row in reader]

        for item in CsvItem:

            if item[0]=='FeedCode':

                continue

            if item[0] not in codeList:

                codeList.append(item[0])

                #print CsvItem[15]

            if len(item)<=5:

                print fileName

                print item

                print '++++++++++++++++++++++++++++++'

            #if afterOneMin(item[3])==0:

            #    print item[3]

            #    continue

            if item[5]==zhengshang and item[2]=='Sent':

                zhengshangSent.append(int(item[4]))

            elif item[5]==zhengshang and item[2]=='Acked':

                zhengshangAcked.append(int(item[4]))

            elif item[5]==dashang and item[2]=='Sent':

                 dashangSent.append(int(item[4]))

            elif item[5]==dashang and item[2]=='Acked':

                 dashangAcked.append(int(item[4]))

            elif item[5]==shangqi and item[2]=='Sent':

                 shangqiSent.append(int(item[4]))

                 if int(item[4])>=600000:

                     print "------------"

                     print item

            elif item[5]==shangqi and item[2]=='Acked':

                 shangqiAcked.append(int(item[4]))

            elif item[5]==zhongjin and item[2]=='Sent':

                 zhongjinSent.append(int(item[4]))

            elif item[5]==zhongjin and item[2]=='Acked':

                 zhongjinAcked.append(int(item[4]))

            else:

                 print "wrong info"

                 print item

        if mean(shangqiSent)>420000:

            print sum(shangqiSent)

            print len(shangqiSent)

            print item

            print fileName

            print shangqiSent

        zhengshangSentMedian=median(zhengshangSent)

        zhengshangSentMean=mean(zhengshangSent)

        zhengshangAckedMedian=median(zhengshangAcked)

        zhengshangAckedMean=mean(zhengshangAcked)

        dashangSentMedian=median(dashangSent)

        dashangSentMean=mean(dashangSent)

        dashangAckedMedian=median(dashangAcked)

        dashangAckedMean=mean(dashangAcked)

        shangqiSentMedian=median(shangqiSent)

        shangqiSentMean=mean(shangqiSent)

        shangqiAckedMedian=median(shangqiAcked)

        shangqiAckedMean=mean(shangqiAcked)

        zhongjinSentMedian=median(zhongjinSent)

        zhongjinSentMean=mean(zhongjinSent)

        zhongjinAckedMedian=median(zhongjinAcked)

        zhongjinAckedMean=mean(zhongjinAcked)

        #if mean(shangqiSent)>70:

        #    print '================================'

        #    print fileName

        #print codeList

        '''

        for listItem in codeList:

            itemSentList=[]

            itemAckedList=[]

            for item in CsvItem:

                if item[0]==listItem and item[2]=='Sent':

                    itemSentList.append(int(item[4]))

                    exchange=item[5]

                elif  item[0]==listItem and item[2]=='Acked':

                    itemAckedList.append(int(item[4]))

            #print itemSentList

            itemMaxSent=max(itemSentList)

            itemMinSent=min(itemSentList)

            itemAvgSent=sum(itemSentList)/len(itemSentList)

            itemMaxAcked=max(itemAckedList)

            itemMinAcked=min(itemAckedList)

            itemAvgAcked=sum(itemAckedList)/len(itemAckedList)

            SentMedian=median(itemSentList)

            AckedMedian=median(itemAckedList)

            msg=[]

            msg.append(listItem)         #0

            msg.append("2015/01/14")   #1

            msg.append(itemMaxSent)     #2

            msg.append(itemMinSent)     #3

            msg.append(itemAvgSent)     #4

            msg.append(SentMedian)     #5

            msg.append(itemMaxAcked)    #6

            msg.append(itemMinAcked)     #7

            msg.append(itemAvgAcked)     #8

            msg.append(AckedMedian)      #9

            msg.append(exchange)          #10

            if len(msg)>15:

               print "------------------------------"

               print msg

            writer1.writerow(msg)

         '''

        zhengshangSentMedianAll.append(zhengshangSentMedian)

        zhengshangSentMeanAll.append(zhengshangSentMean)

        zhengshangAckedMedianAll.append(zhengshangAckedMedian)

        zhengshangAckedMeanAll.append(zhengshangAckedMean)

        dashangSentMedianAll.append(dashangSentMedian)

        dashangSentMeanAll.append(dashangSentMean)

        dashangAckedMedianAll.append(dashangAckedMedian)

        dashangAckedMeanAll.append(dashangAckedMean)

        shangqiSentMedianAll.append(shangqiSentMedian)

        shangqiSentMeanAll.append(shangqiSentMean)

        shangqiAckedMedianAll.append(shangqiAckedMedian)

        shangqiAckedMeanAll.append(shangqiAckedMean)

        zhongjinSentMedianAll.append(zhongjinSentMedian)

        zhongjinSentMeanAll.append(zhongjinSentMean)

        zhongjinAckedMedianAll.append(zhongjinAckedMedian)

        zhongjinAckedMeanAll.append(zhongjinAckedMean)

plt.figure(1)

plt.figure(2)

plt.figure(3)

plt.figure(4)

plt.figure(1)

plt.title('SentMean r-zhengshang b-dashang,green-shangqi grey-zhongjin')

plt.plot(range(1,len(zhengshangSentMeanAll)+1),zhengshangSentMeanAll,'r')

plt.plot(range(1,len(dashangSentMeanAll)+1),dashangSentMeanAll,'b')

plt.plot(range(1,len(shangqiSentMeanAll)+1),shangqiSentMeanAll,'g')

plt.plot(range(1,len(zhongjinSentMeanAll)+1),zhongjinSentMeanAll,'grey')

plt.savefig('/nethome/dehua.li/orderlifeMyWork/xingzheng/data_noTPO_in10minutes/SentMean.png')

plt.figure(2)

plt.title('SentMedian r-zhengshang b-dashang,green-shangqi grey-zhongjin')

plt.plot(range(1,len(zhengshangSentMedianAll)+1),zhengshangSentMedianAll,'r')

plt.plot(range(1,len(dashangSentMedianAll)+1),dashangSentMedianAll,'b')

plt.plot(range(1,len(shangqiSentMedianAll)+1),shangqiSentMedianAll,'g')

plt.plot(range(1,len(zhongjinSentMedianAll)+1),zhongjinSentMedianAll,'grey')

plt.savefig('/nethome/dehua.li/orderlifeMyWork/xingzheng/data_noTPO_in10minutes/SentMedian.png')

plt.figure(3)

plt.title('AckedMean r-zhengshang b-dashang,green-shangqi grey-zhongjin')

plt.plot(range(1,len(zhengshangAckedMeanAll)+1),zhengshangAckedMeanAll,'r')

plt.plot(range(1,len(dashangAckedMeanAll)+1),dashangAckedMeanAll,'b')

plt.plot(range(1,len(shangqiAckedMeanAll)+1),shangqiAckedMeanAll,'g')

plt.plot(range(1,len(zhongjinAckedMeanAll)+1),zhongjinAckedMeanAll,'grey')

plt.savefig('/nethome/dehua.li/orderlifeMyWork/xingzheng/data_noTPO_in10minutes/AckedMean.png')

plt.figure(4)

plt.title('AckedMedian r-zhengshang b-dashang,green-shangqi grey-zhongjin')

plt.plot(range(1,len(zhengshangAckedMedianAll)+1),zhengshangAckedMedianAll,'r')

plt.plot(range(1,len(dashangAckedMedianAll)+1),dashangAckedMedianAll,'b')

plt.plot(range(1,len(shangqiAckedMedianAll)+1),shangqiAckedMedianAll,'g')

plt.plot(range(1,len(zhongjinAckedMedianAll)+1),zhongjinAckedMedianAll,'grey')

plt.savefig('/nethome/dehua.li/orderlifeMyWork/xingzheng/data_noTPO_in10minutes/AckedMedian.png')

plt.show()

print 'over'

希望本文所述对大家的Python程序设计有所帮助。

Python 相关文章推荐
python应用程序在windows下不出现cmd窗口的办法
May 29 Python
Python中用post、get方式提交数据的方法示例
Sep 22 Python
python 随机生成10位数密码的实现代码
Jun 27 Python
VPS CENTOS 上配置python,mysql,nginx,uwsgi,django的方法详解
Jul 01 Python
python opencv 二值化 计算白色像素点的实例
Jul 03 Python
使用Windows批处理和WMI设置Python的环境变量方法
Aug 14 Python
PyCharm2018 安装及破解方法实现步骤
Sep 09 Python
pygame实现成语填空游戏
Oct 29 Python
Python如何使用paramiko模块连接linux
Mar 18 Python
Python多线程通信queue队列用法实例分析
Mar 24 Python
使用Keras实现Tensor的相乘和相加代码
Jun 18 Python
Python类方法总结讲解
Jul 26 Python
python模拟鼠标拖动操作的方法
Mar 11 #Python
Python创建系统目录的方法
Mar 11 #Python
Python实现从订阅源下载图片的方法
Mar 11 #Python
Python获取网页上图片下载地址的方法
Mar 11 #Python
Python实现从百度API获取天气的方法
Mar 11 #Python
Python计算回文数的方法
Mar 11 #Python
Python用GET方法上传文件
Mar 10 #Python
You might like
php获取服务器信息的实现代码
2013/02/04 PHP
PHP使用mysql_fetch_row查询获得数据行列表的方法
2015/03/18 PHP
浅谈PHP Cookie处理函数
2016/06/10 PHP
PHP中trait使用方法详细介绍
2017/05/21 PHP
PHP实现在对象之外访问其私有属性private及保护属性protected的方法
2017/11/20 PHP
web css实现整站样式互相切换
2013/10/29 Javascript
js创建对象的方法汇总
2016/01/07 Javascript
jQuery插件formValidator实现表单验证
2016/05/23 Javascript
JavaScript中的各种操作符使用总结
2016/05/26 Javascript
详解js中常规日期格式处理、月历渲染和倒计时函数
2016/12/28 Javascript
JavaScript实现经典排序算法之选择排序
2016/12/28 Javascript
js实现手机拍照上传功能
2017/01/17 Javascript
详解webpack和webpack-simple中如何引入css文件
2017/06/28 Javascript
解决bootstrap-select 动态加载数据不显示的问题
2018/08/10 Javascript
快速解决angularJS中用post方法时后台拿不到值的问题
2018/08/14 Javascript
vue-router动态设置页面title的实例讲解
2018/08/30 Javascript
解决vue热替换失效的根本原因
2018/09/19 Javascript
iview Upload组件多个文件上传的示例代码
2018/09/30 Javascript
vue实现搜索功能
2019/05/28 Javascript
vue.js 子组件无法获取父组件store值的解决方式
2019/11/08 Javascript
使用TS来编写express服务器的方法步骤
2020/10/29 Javascript
[09:13]2014DOTA2国际邀请赛 中国区预选赛coser表演
2014/05/23 DOTA
[03:53]2016国际邀请赛中国区预选赛第三日TOP10精彩集锦
2016/06/29 DOTA
python函数缺省值与引用学习笔记分享
2013/02/10 Python
Python3爬虫学习之MySQL数据库存储爬取的信息详解
2018/12/12 Python
对python:循环定义多个变量的实例详解
2019/01/20 Python
Python数据库小程序源代码
2019/09/15 Python
python3 中时间戳、时间、日期的转换和加减操作
2020/07/14 Python
PyTorch实现重写/改写Dataset并载入Dataloader
2020/07/14 Python
日本小田急百货官网:Odakyu
2018/07/19 全球购物
洲际酒店集团英国官网:IHG英国
2019/07/10 全球购物
党员个人整改方案及措施
2014/10/25 职场文书
2014年乡镇安全生产工作总结
2014/12/02 职场文书
满月酒邀请函
2015/01/30 职场文书
简爱电影观后感
2015/06/10 职场文书
详解overflow:hidden的作用(溢出隐藏、清除浮动、解决外边距塌陷)
2021/07/01 HTML / CSS