python制作mysql数据迁移脚本


Posted in Python onJanuary 01, 2019

用python写了个数据迁移脚本,主要是利用从库将大的静态表导出表空间,载导入到目标实例中。

#!/usr/bin/env python3
#-*- coding:utf8 -*-
#author:zhanbin.liu
#!!!!!DB必须同版本
#python3环境  pip3 install pymysql paramiko

import os
#from pathlib import Path
import sys
import pymysql
import paramiko

#每次只能迁移一个DB下的表,到指定DB
#GRANT SELECT, CREATE, RELOAD, ALTER, LOCK TABLES ON *.* TO 'data_migration'@'192.168.%' IDENTIFIED BY 'data_migration@123';
tables='sqlauto_cluster,sqlauto_user'    #以,分割的字符串,如a,b,c
tableList = tables.split(',')
sourceIp = '192.168.1.101'
sourceDataBase = '/data/mysql/3306/data'
sourceDbName = 'inception_web'
sourceDataDir = os.path.join(sourceDataBase,sourceDbName)
desIp = '192.168.1.102'
desDataBase = '/data/mysql/3306/data'
desDbName = 'inception_web'
desDataDir = os.path.join(desDataBase,desDbName)

# for table in tableList:
#   desFile = Path("%s/%s.ibd" %(desDataDir,table))
#   print(desFile)
#   if desFile.is_file():
#     print("ok")
#   else:
#     print("no")

comUser = 'data_migration'
comPwd = 'data_migration@123'
comPort = 3306

client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())

def table_judge():
  print("table_judge")
  sourceTableExist = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')
  desTableExist = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')
  sourceTables = []
  desTables = []
  cursor_source = sourceTableExist.cursor()
  cursor_des = desTableExist.cursor()

  for table in tableList:
    #print(table)
    cursor_source.execute("select TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s';" % (sourceDbName,table))
    sourceTable_tmp = cursor_source.fetchall()
    cursor_des.execute("select TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s';" % (desDbName,table))
    desTable_tmp = cursor_des.fetchall()
    #print(desTable_tmp)
    if sourceTable_tmp is ():
      sourceTables.append(table)
    if desTable_tmp is not ():
      desTables.append(desTable_tmp[0][0])
  sourceTableExist.close()
  desTableExist.close()

  s=d=0
  if sourceTables != []:
    print('迁移源不存在将要迁移的表:',sourceIp,sourceDbName, sourceTables,' 请检查')
    s=1
  if desTables != []:
    print('目标库存在将要迁移的表:',desIp,desDbName,desTables,' 请移除')
    d=1
  if s == 1 or d == 1:
    sys.exit()

def data_sync():
  print('data_sync')
  db_source = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')
  db_des = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')
  cursor_db_source = db_source.cursor()
  cursor_db_des = db_des.cursor()

  for table in tableList:
    print("正在同步表:",table)
    cursor_db_source.execute("show create table %s;" % (table))
    createTableSQL = cursor_db_source.fetchall()[0][1]
    print(createTableSQL)
    try:
      cursor_db_des.execute(createTableSQL)
    except Exception as error:
      print(error)
    cursor_db_source.execute("flush table %s with read lock;" % (table))
    cursor_db_des.execute("alter table %s discard tablespace;" % (table))

    client.connect(sourceIp, 22, 'root')
    stdin1, stdout1, stderr1 = client.exec_command("scp %s %s:%s " % (sourceDataDir+"/"+table+".ibd", desIp, desDataDir))
    stdin2, stdout2, stderr2 = client.exec_command("scp %s %s:%s " % (sourceDataDir+"/"+table+".cfg", desIp, desDataDir))
    a_e_1 = stderr1.readlines()
    a_e_2 = stderr2.readlines()
    if a_e_1 != [] or a_e_2 != []:
      print(a_e_1,a_e_2)
      sys.exit()
    client.close()

    client.connect(desIp, 22, 'root')
    stdin3, stdout3, stderr3 = client.exec_command("chown -R mysql.mysql %s*" % (desDataDir+"/"+table))
    a_e_3 = stderr3.readlines()
    if a_e_3 != []:
      print(a_e_1, a_e_2)
      sys.exit()
    client.close()
    #cursor_db_source.execute("select sleep(10);")
    cursor_db_source.execute("unlock tables;")
    cursor_db_des.execute("alter table %s import tablespace;" % (table))
    print("同步完成")

  cursor_db_source.close()
  cursor_db_des.close()

def data_checksum():
  print('data_checksum')
  db_source = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')
  db_des = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')
  cursor_db_source = db_source.cursor()
  cursor_db_des = db_des.cursor()

  for table in tableList:
    print("正在校验表:", table)
    cursor_db_source.execute("checksum table %s;" % (table))
    ck_s = cursor_db_source.fetchall()[0][1]
    cursor_db_des.execute("checksum table %s;" % (table))
    ck_d = cursor_db_des.fetchall()[0][1]
    if ck_s != ck_d:
      print("表不一致:",table)
    else:
      print("表一致:",table)

  cursor_db_source.close()
  cursor_db_des.close()

if __name__ == "__main__":
  table_judge()
  data_sync()
  data_checksum()
  print('haha')
Python 相关文章推荐
Python获取当前时间的方法
Jan 14 Python
Python中list列表的一些进阶使用方法介绍
Aug 15 Python
理解python中生成器用法
Dec 20 Python
Python使用matplotlib填充图形指定区域代码示例
Jan 16 Python
Python发送邮件测试报告操作实例详解
Dec 08 Python
Python实现字符串匹配的KMP算法
Apr 04 Python
Python日期时间Time模块实例详解
Apr 15 Python
python and or用法详解
Jun 26 Python
python3 pathlib库Path类方法总结
Dec 26 Python
pycharm下pyqt4安装及环境配置的教程
Apr 24 Python
Python如何获取文件路径/目录
Sep 22 Python
python爬虫爬取淘宝商品比价(附淘宝反爬虫机制解决小办法)
Dec 03 Python
在python中将字符串转为json对象并取值的方法
Dec 31 #Python
对python中Json与object转化的方法详解
Dec 31 #Python
python使用zip将list转为json的方法
Dec 31 #Python
python 获取utc时间转化为本地时间的方法
Dec 31 #Python
python 实现UTC时间加减的方法
Dec 31 #Python
Python从单元素字典中获取key和value的实例
Dec 31 #Python
对Python 两大环境管理神器 pyenv 和 virtualenv详解
Dec 31 #Python
You might like
JSON在PHP中的应用介绍
2012/09/08 PHP
解决PHP mysql_query执行超时(Fatal error: Maximum execution time …)
2013/07/03 PHP
web server使用php生成web页面的三种方法总结
2013/10/28 PHP
PHP中提问频率最高的11个面试题和答案
2014/09/02 PHP
thinkphp文件引用与分支结构用法实例
2014/11/26 PHP
PHP扩展模块memcached长连接使用方法分析
2014/12/24 PHP
PHP常见错误提示含义解释(实用!值得收藏)
2016/04/25 PHP
Yii2搭建后台并实现rbac权限控制完整实例教程
2016/04/28 PHP
[原创]smarty简单模板变量输出方法
2016/07/09 PHP
Laravel5中防止XSS跨站攻击的方法
2016/10/10 PHP
JQuery 无废话系列教程(二) jquery实战篇上
2009/06/23 Javascript
详解JS面向对象编程
2016/01/24 Javascript
JS弹出新窗口被拦截的解决方法
2016/08/09 Javascript
纯css下拉菜单 无需js
2016/08/15 Javascript
js实现按座位号抽奖
2017/04/05 Javascript
给Easyui-Datebox设置隐藏或者不可用的解决方法
2017/05/26 Javascript
JS自定义函数实现时间戳转换成date的方法示例
2017/08/27 Javascript
微信小程序wx.uploadfile 本地文件转base64的实现代码
2018/06/28 Javascript
JS使用new操作符创建对象的方法分析
2019/05/30 Javascript
Vue请求java服务端并返回数据代码实例
2019/11/28 Javascript
Vue生命周期activated之返回上一页不重新请求数据操作
2020/07/26 Javascript
JavaScript字符串转数字的简单实现方法
2020/11/27 Javascript
Python打印斐波拉契数列实例
2015/07/07 Python
python读取oracle函数返回值
2016/07/18 Python
Python SQLite3简介
2018/02/22 Python
Python入门学习指南分享
2018/04/11 Python
pycharm设置注释颜色的方法
2018/05/23 Python
详解Python Opencv和PIL读取图像文件的差别
2019/12/27 Python
html Table 表头固定的实现
2019/01/22 HTML / CSS
在加拿大在线租赁和购买电子游戏:Game Access
2019/09/02 全球购物
Java基础知识面试要点
2016/07/29 面试题
物流管理应届生求职信
2013/11/07 职场文书
会计专业个人求职信范文
2014/01/08 职场文书
网络宣传方案
2014/03/15 职场文书
优秀的2014年两会精神解读
2014/03/17 职场文书
大一学生个人总结
2015/02/15 职场文书