python制作mysql数据迁移脚本


Posted in Python onJanuary 01, 2019

用python写了个数据迁移脚本,主要是利用从库将大的静态表导出表空间,载导入到目标实例中。

#!/usr/bin/env python3
#-*- coding:utf8 -*-
#author:zhanbin.liu
#!!!!!DB必须同版本
#python3环境  pip3 install pymysql paramiko

import os
#from pathlib import Path
import sys
import pymysql
import paramiko

#每次只能迁移一个DB下的表,到指定DB
#GRANT SELECT, CREATE, RELOAD, ALTER, LOCK TABLES ON *.* TO 'data_migration'@'192.168.%' IDENTIFIED BY 'data_migration@123';
tables='sqlauto_cluster,sqlauto_user'    #以,分割的字符串,如a,b,c
tableList = tables.split(',')
sourceIp = '192.168.1.101'
sourceDataBase = '/data/mysql/3306/data'
sourceDbName = 'inception_web'
sourceDataDir = os.path.join(sourceDataBase,sourceDbName)
desIp = '192.168.1.102'
desDataBase = '/data/mysql/3306/data'
desDbName = 'inception_web'
desDataDir = os.path.join(desDataBase,desDbName)

# for table in tableList:
#   desFile = Path("%s/%s.ibd" %(desDataDir,table))
#   print(desFile)
#   if desFile.is_file():
#     print("ok")
#   else:
#     print("no")

comUser = 'data_migration'
comPwd = 'data_migration@123'
comPort = 3306

client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())

def table_judge():
  print("table_judge")
  sourceTableExist = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')
  desTableExist = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')
  sourceTables = []
  desTables = []
  cursor_source = sourceTableExist.cursor()
  cursor_des = desTableExist.cursor()

  for table in tableList:
    #print(table)
    cursor_source.execute("select TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s';" % (sourceDbName,table))
    sourceTable_tmp = cursor_source.fetchall()
    cursor_des.execute("select TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s';" % (desDbName,table))
    desTable_tmp = cursor_des.fetchall()
    #print(desTable_tmp)
    if sourceTable_tmp is ():
      sourceTables.append(table)
    if desTable_tmp is not ():
      desTables.append(desTable_tmp[0][0])
  sourceTableExist.close()
  desTableExist.close()

  s=d=0
  if sourceTables != []:
    print('迁移源不存在将要迁移的表:',sourceIp,sourceDbName, sourceTables,' 请检查')
    s=1
  if desTables != []:
    print('目标库存在将要迁移的表:',desIp,desDbName,desTables,' 请移除')
    d=1
  if s == 1 or d == 1:
    sys.exit()

def data_sync():
  print('data_sync')
  db_source = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')
  db_des = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')
  cursor_db_source = db_source.cursor()
  cursor_db_des = db_des.cursor()

  for table in tableList:
    print("正在同步表:",table)
    cursor_db_source.execute("show create table %s;" % (table))
    createTableSQL = cursor_db_source.fetchall()[0][1]
    print(createTableSQL)
    try:
      cursor_db_des.execute(createTableSQL)
    except Exception as error:
      print(error)
    cursor_db_source.execute("flush table %s with read lock;" % (table))
    cursor_db_des.execute("alter table %s discard tablespace;" % (table))

    client.connect(sourceIp, 22, 'root')
    stdin1, stdout1, stderr1 = client.exec_command("scp %s %s:%s " % (sourceDataDir+"/"+table+".ibd", desIp, desDataDir))
    stdin2, stdout2, stderr2 = client.exec_command("scp %s %s:%s " % (sourceDataDir+"/"+table+".cfg", desIp, desDataDir))
    a_e_1 = stderr1.readlines()
    a_e_2 = stderr2.readlines()
    if a_e_1 != [] or a_e_2 != []:
      print(a_e_1,a_e_2)
      sys.exit()
    client.close()

    client.connect(desIp, 22, 'root')
    stdin3, stdout3, stderr3 = client.exec_command("chown -R mysql.mysql %s*" % (desDataDir+"/"+table))
    a_e_3 = stderr3.readlines()
    if a_e_3 != []:
      print(a_e_1, a_e_2)
      sys.exit()
    client.close()
    #cursor_db_source.execute("select sleep(10);")
    cursor_db_source.execute("unlock tables;")
    cursor_db_des.execute("alter table %s import tablespace;" % (table))
    print("同步完成")

  cursor_db_source.close()
  cursor_db_des.close()

def data_checksum():
  print('data_checksum')
  db_source = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')
  db_des = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')
  cursor_db_source = db_source.cursor()
  cursor_db_des = db_des.cursor()

  for table in tableList:
    print("正在校验表:", table)
    cursor_db_source.execute("checksum table %s;" % (table))
    ck_s = cursor_db_source.fetchall()[0][1]
    cursor_db_des.execute("checksum table %s;" % (table))
    ck_d = cursor_db_des.fetchall()[0][1]
    if ck_s != ck_d:
      print("表不一致:",table)
    else:
      print("表一致:",table)

  cursor_db_source.close()
  cursor_db_des.close()

if __name__ == "__main__":
  table_judge()
  data_sync()
  data_checksum()
  print('haha')
Python 相关文章推荐
用smtplib和email封装python发送邮件模块类分享
Feb 17 Python
Python实现豆瓣图片下载的方法
May 25 Python
Django框架中render_to_response()函数的使用方法
Jul 16 Python
Python基于有道实现英汉字典功能
Jul 25 Python
Python正则表达式知识汇总
Sep 22 Python
浅谈Python中的bs4基础
Oct 21 Python
python图形工具turtle绘制国际象棋棋盘
May 23 Python
python打包exe开机自动启动的实例(windows)
Jun 28 Python
对Django中内置的User模型实例详解
Aug 16 Python
利用python中集合的唯一性实现去重
Feb 11 Python
matlab灰度图像调整及imadjust函数的用法详解
Feb 27 Python
python实现猜单词游戏
May 22 Python
在python中将字符串转为json对象并取值的方法
Dec 31 #Python
对python中Json与object转化的方法详解
Dec 31 #Python
python使用zip将list转为json的方法
Dec 31 #Python
python 获取utc时间转化为本地时间的方法
Dec 31 #Python
python 实现UTC时间加减的方法
Dec 31 #Python
Python从单元素字典中获取key和value的实例
Dec 31 #Python
对Python 两大环境管理神器 pyenv 和 virtualenv详解
Dec 31 #Python
You might like
php中强制下载文件的代码(解决了IE下中文文件名乱码问题)
2011/05/09 PHP
php 保留字列表
2012/10/04 PHP
PHP+Javascript实现在线拍照功能实例
2015/07/18 PHP
php中define用法实例
2015/07/30 PHP
PHP微信API接口类
2016/08/22 PHP
thinkPHP订单数字提醒功能的实现方法
2016/12/01 PHP
PHP面向对象程序设计OOP继承用法入门示例
2016/12/27 PHP
PHP 二维array转换json的实例讲解
2018/08/21 PHP
JS动态获取当前时间,并写到特定的区域
2013/05/03 Javascript
jquery ajax,ashx,json的用法总结
2014/02/12 Javascript
SeaJS入门教程系列之完整示例(三)
2014/03/03 Javascript
js中的setInterval和setTimeout使用实例
2014/05/09 Javascript
使用JQuery FancyBox插件实现图片展示特效
2015/11/16 Javascript
JQuery validate插件Remote用法大全
2016/05/15 Javascript
vue的常用组件操作方法应用分析
2018/04/13 Javascript
详解离线安装npm包的几种方法
2018/11/25 Javascript
JavaScript的查询机制LHS和RHS解析
2019/08/16 Javascript
微信小程序顶部导航栏可滑动并选中放大
2019/12/05 Javascript
解决VantUI popup 弹窗不弹出或无蒙层的问题
2020/11/03 Javascript
详解vue 组件注册
2020/11/20 Vue.js
python BeautifulSoup使用方法详解
2013/11/21 Python
python 简单的多线程链接实现代码
2016/08/28 Python
用python统计代码行的示例(包括空行和注释)
2018/07/24 Python
对python PLT中的image和skimage处理图片方法详解
2019/01/10 Python
详解Python中的内建函数,可迭代对象,迭代器
2019/04/29 Python
8种用Python实现线性回归的方法对比详解
2019/07/10 Python
对pytorch中的梯度更新方法详解
2019/08/20 Python
推荐10个CSS3 制作的创意下拉菜单效果
2014/02/11 HTML / CSS
html5 input属性使用示例
2013/06/28 HTML / CSS
Lampegiganten丹麦:欧洲领先的照明网上商店
2018/04/25 全球购物
怎样声明接口
2014/09/19 面试题
如何利用cmp命令比较文件
2013/09/23 面试题
国企干部对照检查材料
2014/08/22 职场文书
幼师中班个人总结
2015/02/12 职场文书
开会通知
2015/04/20 职场文书
逃出克隆岛观后感
2015/06/09 职场文书