首页
所有分类
TAGS
文字工具 EMOJI BIBLE
编程 Python

python处理csv数据的方法

Posted in Python onMarch 11, 2015

本文实例讲述了python处理csv数据的方法。分享给大家供大家参考。具体如下：

Python代码：

#coding=utf-8

__author__ = 'dehua.li'

from datetime import *

import datetime

import csv

import sys

import time

import string

import os

import os.path

import pylab as plt

rootdir='/nethome/dehua.li/orderlifeCycleData/xingzheng'

writeFileDir="/nethome/dehua.li/orderlifeMyWork/xingzheng/csv"

heyueFile="/nethome/dehua.li/orderlifeCycleData/heyue_150128.csv"

ms_acked="1"

msg=[]

ex=[]

def getTheDate(date):

    [filenamePart1,filenamePart2]=string.split(filename,'.')

    [filenamePart11,filenamePart12,filenamePart13]=string.split(filenamePart1,'_')

    return filenamePart13

LocalTime=datetime.datetime.fromtimestamp(time.mktime(time.strptime("2014-11-04 20:59:59","%Y-%m-%d %H:%M:%S")))

for parent,dirname,filenames in os.walk(rootdir):

    for filename in filenames:

        fileNameWrite=os.path.join(writeFileDir,filename)

        print fileNameWrite

        csvfile00=open(fileNameWrite,'wb')

        writer1=csv.writer(csvfile00)

        writer1.writerow(['FeedCode','OrderId','Status','LocalTime','Time','Exchange'])

        fileName=os.path.join(parent,filename)

        [filenamePart1,filenamePart2]=string.split(filename,'.')

        [filenamePart11,filenamePart12,filenamePart13]=string.split(filenamePart1,'_')

        #filenamePart11_filenamePart12_filenamePart13.filenamePart2:dongzheng_orderlifeCycleData_20150111.csv

        print fileName

        with open(fileName,'rb') as csvfile:

            reader=csv.reader(csvfile)

            CsvItem=[row for row in reader]

            for item in CsvItem:

                if item[3]=='TPO':

                    #print " filter TPO "

                    continue

                if item[12]=='Sent':

                    [tm_local,ms_local]=string.split(item[15],'.')

                    [tm_localup,ms_localup]=string.split(item[19],'.')

                    LocalTime=datetime.datetime.fromtimestamp(time.mktime(time.strptime(tm_local,"%Y-%m-%d %H:%M:%S")))

                    LocalUpdate=datetime.datetime.fromtimestamp(time.mktime(time.strptime(tm_localup,"%Y-%m-%d %H:%M:%S")))

                    tm=int(((LocalTime-LocalUpdate).seconds))*1000

                    ms_sent=str(int(ms_local)-int(ms_localup)+tm)

                    if int(ms_sent)>10*60*1000:

                        print "ms_sent>600000"

                        continue

                    if(int(ms_local)-int(ms_localup)+tm)<0:

                        print 'wrong1'

                    msg=[]

                    msg.append(item[0])

                    msg.append(item[1])

                    msg.append(item[12])

                    msg.append(item[15])

                    msg.append(ms_sent)

                    with open(heyueFile,'rb') as csvfile1:

                        reader=csv.reader(csvfile1)

                        CsvItem=[row for row in reader]

                        for Item in CsvItem:

                            if Item[1]==item[0]:

                                msg.append(Item[3])

                                writer1.writerow(msg)

                                #print 'write ok'

                                ex=Item[3]

                                break

                    csvfile1.close()

                    with open(fileName,'rb') as csvfile22:

                        reader=csv.reader(csvfile22)

                        CsvItem2=[row for row in reader]

                        for item_ in CsvItem2:

                            if item_[12]=='Acked' and item_[1]==item[1]:

                                [tm_local2,ms_local2]=string.split(item_[15],'.')

                                LocalTime2=datetime.datetime.fromtimestamp(time.mktime(time.strptime(tm_local2,"%Y-%m-%d %H:%M:%S")))

                                tm2=int(((LocalTime2-LocalTime).seconds))*1000

                                ms_acked=str(int(ms_local2)-int(ms_local)+tm2)

                                if int(ms_acked)>10*60*1000:

                                    print "MSacked>600000"

                                    continue

                                msg=[]

                                msg.append(item_[0])

                                msg.append(item_[1])

                                msg.append(item_[12])

                                msg.append(item_[15])

                                msg.append(ms_acked)

                    with open(heyueFile,'rb') as csvfile111:

                        reader=csv.reader(csvfile111)

                        CsvItem=[row for row in reader]

                        for Item in CsvItem:

                            if Item[1]==item[0]:

                                msg.append(Item[3])

                                writer1.writerow(msg)

                                #print 'write ok'

                                break

                                #print "write ok"

                    csvfile22.close()

        csvfile.close()

        csvfile00.close()
#coding=utf-8

#__author__ = 'dehua.li'

from datetime import *

import datetime

import csv

import sys

import time

import string

import os

import os.path

import pylab as plt

def median(lst):

    even = (0 if len(lst) % 2 else 1) + 1

    half = (len(lst) - 1) / 2

    return sum(sorted(lst)[half:half + even]) / float(even)

def mean(lst):

    if len(lst)==0:

        return 0

    return sum(lst)/len(lst)

nightLine="21:01:00"

morningLine="09:01:00"

def getTheDate(date):

    [filenamePart1,filenamePart2]=string.split(filename,'.')

    [filenamePart11,filenamePart12,filenamePart13]=string.split(filenamePart1,'_')

    return filenamePart13

def afterOneMin(time):

    [tm_local,ms_local]=string.split(time,'.')

    [ymd,hms]=string.split(tm_local,' ')

    flag=0

    if hms>"21:01:00":

        flag=1

    elif hms>"09:01:00" and hms<"20:00:00":

        flag=1

    elif hms>"00:00:00" and hms<"05:00:00":

        flag=1

    return flag

rootdir="/nethome/dehua.li/orderlifeMyWork/xingzheng/csv"

#csvfileMaxMin = open('e:\dehua.li\csv\__xingzhenMaxMin.csv','wb')

#writer1 = csv.writer(csvfileMaxMin)

#writer1.writerow(['FeedCode','date','SentMaxTime','SentMaxLocalTime','SentMinTime','SentMinLocalTime','SentMeanTime','SentMedian','AckedMaxTime','AckedMaxLocalTime','AckedMinTime','AckedMinLocalTime','AckedMeanTime','AckedMedianTime','Exchange'])

#writer1.writerow(['FeedCode','date','SentMaxTime','SentMinTime','SentMeanTime','SentMedian','AckedMaxTime','AckedMinTime','AckedMeanTime','AckedMedianTime','Exchange'])

msg=[]

codeList=list()

orderList=list()

itemSentList=[]

itemAckedList=[]

feedCode=[]

exchange=[]

zhengshangSentMedian=0

zhengshangSentMean=0

zhengshangAckedMedian=0

zhengshangAckedMean=0

dashangSentMedian=0

dashangSentMean=0

dashangAckedMedian=0

dashangAckedMean=0

shangqiSentMedian=0

shangqiSentMean=0

shangqiAckedMedian=0

shangqiAckedMean=0

zhongjinSentMedian=0

zhongjinSentMean=0

zhongjinAckedMedian=0

zhongjinAckedMean=0

zhengshangSent=[]

zhengshangAcked=[]

dashangSent=[]

dashangAcked=[]

shangqiSent=[]

shangqiAcked=[]

zhongjinSent=[]

zhongjinAcked=[]

zhengshangSentMedianAll=[]

zhengshangSentMeanAll=[]

zhengshangAckedMedianAll=[]

zhengshangAckedMeanAll=[]

dashangSentMedianAll=[]

dashangSentMeanAll=[]

dashangAckedMedianAll=[]

dashangAckedMeanAll=[]

shangqiSentMedianAll=[]

shangqiSentMeanAll=[]

shangqiAckedMedianAll=[]

shangqiAckedMeanAll=[]

zhongjinSentMedianAll=[]

zhongjinSentMeanAll=[]

zhongjinAckedMedianAll=[]

zhongjinAckedMeanAll=[]

zhengshang='0'

dashang='0'

shangqi='0'

zhongjin='0'

with open('/nethome/dehua.li/orderlifeCycleData/heyue_150128.csv','rb') as csvfile:

    reader=csv.reader(csvfile)

    csvItem=[row for row in reader]

    zhengshang=csvItem[300][3]

    dashang=csvItem[5][3]

    shangqi=csvItem[165][3]

    zhongjin=csvItem[435][3]

    #for item in csvItem:

    #    if item[3]==zhengshang:

     #       print item

for parent,dirname,filenames in os.walk(rootdir):

    for filename in filenames:

        fileName=os.path.join(rootdir,filename)

        csvfile1=open(fileName,'rb')

        reader=csv.reader(csvfile1)

        CsvItem=[row for row in reader]

        for item in CsvItem:

            if item[0]=='FeedCode':

                continue

            if item[0] not in codeList:

                codeList.append(item[0])

                #print CsvItem[15]

            if len(item)<=5:

                print fileName

                print item

                print '++++++++++++++++++++++++++++++'

            #if afterOneMin(item[3])==0:

            #    print item[3]

            #    continue

            if item[5]==zhengshang and item[2]=='Sent':

                zhengshangSent.append(int(item[4]))

            elif item[5]==zhengshang and item[2]=='Acked':

                zhengshangAcked.append(int(item[4]))

            elif item[5]==dashang and item[2]=='Sent':

                 dashangSent.append(int(item[4]))

            elif item[5]==dashang and item[2]=='Acked':

                 dashangAcked.append(int(item[4]))

            elif item[5]==shangqi and item[2]=='Sent':

                 shangqiSent.append(int(item[4]))

                 if int(item[4])>=600000:

                     print "------------"

                     print item

            elif item[5]==shangqi and item[2]=='Acked':

                 shangqiAcked.append(int(item[4]))

            elif item[5]==zhongjin and item[2]=='Sent':

                 zhongjinSent.append(int(item[4]))

            elif item[5]==zhongjin and item[2]=='Acked':

                 zhongjinAcked.append(int(item[4]))

            else:

                 print "wrong info"

                 print item

        if mean(shangqiSent)>420000:

            print sum(shangqiSent)

            print len(shangqiSent)

            print item

            print fileName

            print shangqiSent

        zhengshangSentMedian=median(zhengshangSent)

        zhengshangSentMean=mean(zhengshangSent)

        zhengshangAckedMedian=median(zhengshangAcked)

        zhengshangAckedMean=mean(zhengshangAcked)

        dashangSentMedian=median(dashangSent)

        dashangSentMean=mean(dashangSent)

        dashangAckedMedian=median(dashangAcked)

        dashangAckedMean=mean(dashangAcked)

        shangqiSentMedian=median(shangqiSent)

        shangqiSentMean=mean(shangqiSent)

        shangqiAckedMedian=median(shangqiAcked)

        shangqiAckedMean=mean(shangqiAcked)

        zhongjinSentMedian=median(zhongjinSent)

        zhongjinSentMean=mean(zhongjinSent)

        zhongjinAckedMedian=median(zhongjinAcked)

        zhongjinAckedMean=mean(zhongjinAcked)

        #if mean(shangqiSent)>70:

        #    print '================================'

        #    print fileName

        #print codeList

        '''

        for listItem in codeList:

            itemSentList=[]

            itemAckedList=[]

            for item in CsvItem:

                if item[0]==listItem and item[2]=='Sent':

                    itemSentList.append(int(item[4]))

                    exchange=item[5]

                elif  item[0]==listItem and item[2]=='Acked':

                    itemAckedList.append(int(item[4]))

            #print itemSentList

            itemMaxSent=max(itemSentList)

            itemMinSent=min(itemSentList)

            itemAvgSent=sum(itemSentList)/len(itemSentList)

            itemMaxAcked=max(itemAckedList)

            itemMinAcked=min(itemAckedList)

            itemAvgAcked=sum(itemAckedList)/len(itemAckedList)

            SentMedian=median(itemSentList)

            AckedMedian=median(itemAckedList)

            msg=[]

            msg.append(listItem)         #0

            msg.append("2015/01/14")   #1

            msg.append(itemMaxSent)     #2

            msg.append(itemMinSent)     #3

            msg.append(itemAvgSent)     #4

            msg.append(SentMedian)     #5

            msg.append(itemMaxAcked)    #6

            msg.append(itemMinAcked)     #7

            msg.append(itemAvgAcked)     #8

            msg.append(AckedMedian)      #9

            msg.append(exchange)          #10

            if len(msg)>15:

               print "------------------------------"

               print msg

            writer1.writerow(msg)

         '''

        zhengshangSentMedianAll.append(zhengshangSentMedian)

        zhengshangSentMeanAll.append(zhengshangSentMean)

        zhengshangAckedMedianAll.append(zhengshangAckedMedian)

        zhengshangAckedMeanAll.append(zhengshangAckedMean)

        dashangSentMedianAll.append(dashangSentMedian)

        dashangSentMeanAll.append(dashangSentMean)

        dashangAckedMedianAll.append(dashangAckedMedian)

        dashangAckedMeanAll.append(dashangAckedMean)

        shangqiSentMedianAll.append(shangqiSentMedian)

        shangqiSentMeanAll.append(shangqiSentMean)

        shangqiAckedMedianAll.append(shangqiAckedMedian)

        shangqiAckedMeanAll.append(shangqiAckedMean)

        zhongjinSentMedianAll.append(zhongjinSentMedian)

        zhongjinSentMeanAll.append(zhongjinSentMean)

        zhongjinAckedMedianAll.append(zhongjinAckedMedian)

        zhongjinAckedMeanAll.append(zhongjinAckedMean)

plt.figure(1)

plt.figure(2)

plt.figure(3)

plt.figure(4)

plt.figure(1)

plt.title('SentMean r-zhengshang b-dashang,green-shangqi grey-zhongjin')

plt.plot(range(1,len(zhengshangSentMeanAll)+1),zhengshangSentMeanAll,'r')

plt.plot(range(1,len(dashangSentMeanAll)+1),dashangSentMeanAll,'b')

plt.plot(range(1,len(shangqiSentMeanAll)+1),shangqiSentMeanAll,'g')

plt.plot(range(1,len(zhongjinSentMeanAll)+1),zhongjinSentMeanAll,'grey')

plt.savefig('/nethome/dehua.li/orderlifeMyWork/xingzheng/data_noTPO_in10minutes/SentMean.png')

plt.figure(2)

plt.title('SentMedian r-zhengshang b-dashang,green-shangqi grey-zhongjin')

plt.plot(range(1,len(zhengshangSentMedianAll)+1),zhengshangSentMedianAll,'r')

plt.plot(range(1,len(dashangSentMedianAll)+1),dashangSentMedianAll,'b')

plt.plot(range(1,len(shangqiSentMedianAll)+1),shangqiSentMedianAll,'g')

plt.plot(range(1,len(zhongjinSentMedianAll)+1),zhongjinSentMedianAll,'grey')

plt.savefig('/nethome/dehua.li/orderlifeMyWork/xingzheng/data_noTPO_in10minutes/SentMedian.png')

plt.figure(3)

plt.title('AckedMean r-zhengshang b-dashang,green-shangqi grey-zhongjin')

plt.plot(range(1,len(zhengshangAckedMeanAll)+1),zhengshangAckedMeanAll,'r')

plt.plot(range(1,len(dashangAckedMeanAll)+1),dashangAckedMeanAll,'b')

plt.plot(range(1,len(shangqiAckedMeanAll)+1),shangqiAckedMeanAll,'g')

plt.plot(range(1,len(zhongjinAckedMeanAll)+1),zhongjinAckedMeanAll,'grey')

plt.savefig('/nethome/dehua.li/orderlifeMyWork/xingzheng/data_noTPO_in10minutes/AckedMean.png')

plt.figure(4)

plt.title('AckedMedian r-zhengshang b-dashang,green-shangqi grey-zhongjin')

plt.plot(range(1,len(zhengshangAckedMedianAll)+1),zhengshangAckedMedianAll,'r')

plt.plot(range(1,len(dashangAckedMedianAll)+1),dashangAckedMedianAll,'b')

plt.plot(range(1,len(shangqiAckedMedianAll)+1),shangqiAckedMedianAll,'g')

plt.plot(range(1,len(zhongjinAckedMedianAll)+1),zhongjinAckedMedianAll,'grey')

plt.savefig('/nethome/dehua.li/orderlifeMyWork/xingzheng/data_noTPO_in10minutes/AckedMedian.png')

plt.show()

print 'over'

希望本文所述对大家的Python程序设计有所帮助。

python处理csv数据的方法

- Author -

kirkversion1

声明：登载此文出于传递更多信息之目的，并不意味着赞同其观点或证实其描述。

Python 相关文章推荐
使用Python获取Linux系统的各种信息
Jul 10 Python
python数组复制拷贝的实现方法
Jun 09 Python
Python编程中的文件操作攻略
Oct 16 Python
尝试使用Python多线程抓取代理服务器IP地址的示例
Nov 09 Python
python中matplotlib实现最小二乘法拟合的过程详解
Jul 11 Python
PyQt5每天必学之弹出消息框
Apr 19 Python
Python实现按中文排序的方法示例
Apr 25 Python
python多进程使用及线程池的使用方法代码详解
Oct 24 Python
使用Python和OpenCV检测图像中的物体并将物体裁剪下来
Oct 30 Python
用python按照图像灰度值统计并筛选图片的操作(PIL,shutil,os)
Jun 04 Python
python判断元素是否存在的实例方法
Sep 24 Python
Python中 range | np.arange | np.linspace三者的区别
Mar 22 Python
python模拟鼠标拖动操作的方法
Mar 11 #Python
Python创建系统目录的方法
Mar 11 #Python
Python实现从订阅源下载图片的方法
Mar 11 #Python
Python获取网页上图片下载地址的方法
Mar 11 #Python
Python实现从百度API获取天气的方法
Mar 11 #Python
Python计算回文数的方法
Mar 11 #Python
Python用GET方法上传文件
Mar 10 #Python
Seaborn(1) Xception(1) yolo3(1) tensorflow(2) Inception(2) jenkins(1) 进程假死(1) 逆序(1) 字符串(2) UV(1)
You might like
php下将图片以二进制存入mysql数据库中并显示的实现代码
2010/05/27 PHP
PHP中OpenSSL加密问题整理
2017/12/14 PHP
JavaScript中setAttribute用法介绍
2013/07/20 Javascript
在JavaScript中使用timer示例
2014/05/08 Javascript
jquery实现的3D旋转木马特效代码分享
2015/08/25 Javascript
apply和call方法定义及apply和call方法的区别
2015/11/15 Javascript
一道常被人轻视的web前端常见面试题(JS)
2016/02/15 Javascript
ES6中Symbol类型用法实例详解
2017/04/06 Javascript
angularjs实现搜索的关键字在正文中高亮出来
2017/06/13 Javascript
详解AngularJS ng-class样式切换
2017/06/27 Javascript
详解如何在React组件“外”使用父组件的Props
2018/01/12 Javascript
详解element-ui设置下拉选择切换必填和非必填
2019/06/17 Javascript
jQuery Datatables 动态列+跨列合并实现代码
2020/01/30 jQuery
[38:30]2014 DOTA2国际邀请赛中国区预选赛 LGD-GAMING VS CIS 第一场2
2014/05/24 DOTA
Python实现类继承实例
2014/07/04 Python
跟老齐学Python之list和str比较
2014/09/20 Python
Python的Django框架中从url中捕捉文本的方法
2015/07/20 Python
python使用matplotlib绘制柱状图教程
2017/02/08 Python
python django事务transaction源码分析详解
2017/03/17 Python
Python SQL查询并生成json文件操作示例
2018/08/17 Python
python实现桌面壁纸切换功能
2019/01/21 Python
Python中整数的缓存机制讲解
2019/02/16 Python
django框架防止XSS注入的方法分析
2019/06/21 Python
python3.6生成器yield用法实例分析
2019/08/23 Python
使用Python脚本zabbix自定义key监控oracle连接状态
2019/08/28 Python
3分钟看懂Python后端必须知道的Django的信号机制
2020/07/26 Python
pandas按条件筛选数据的实现
2021/02/20 Python
英国领先的餐饮折扣俱乐部：Gourmet Society
2020/07/26 全球购物
办公室文秘自我鉴定
2013/09/21 职场文书
学校春季防火方案
2014/06/08 职场文书
坚守艰苦奋斗精神坚决反对享乐主义整改措施
2014/09/17 职场文书
委托书的样本
2015/01/28 职场文书
2016国庆节67周年红领巾广播稿
2015/12/18 职场文书
2019生态环境保护倡议书！
2019/07/03 职场文书
HTML中table表格拆分合并(colspan、rowspan)
2021/04/07 HTML / CSS
再谈python_tkinter弹出对话框创建
2022/03/20 Python