DataFrame:通过SparkSql将scala类转为DataFrame的方法


Posted in Python onJanuary 29, 2019

如下所示:

import java.text.DecimalFormat
import com.alibaba.fastjson.JSON
import com.donews.data.AppConfig
import com.typesafe.config.ConfigFactory
import org.apache.spark.sql.types.{StructField, StructType}
import org.apache.spark.sql.{Row, SaveMode, DataFrame, SQLContext}
import org.apache.spark.{SparkConf, SparkContext}
import org.slf4j.LoggerFactory
 
/**
 * Created by silentwolf on 2016/6/3.
 */
 
case class UserTag(SUUID: String,
     MAN: Float,
     WOMAN: Float,
     AGE10_19: Float,
     AGE20_29: Float,
     AGE30_39: Float,
     AGE40_49: Float,
     AGE50_59: Float,
     GAME: Float,
     MOVIE: Float,
     MUSIC: Float,
     ART: Float,
     POLITICS_NEWS: Float,
     FINANCIAL: Float,
     EDUCATION_TRAINING: Float,
     HEALTH_CARE: Float,
     TRAVEL: Float,
     AUTOMOBILE: Float,
     HOUSE_PROPERTY: Float,
     CLOTHING_ACCESSORIES: Float,
     BEAUTY: Float,
     IT: Float,
     BABY_PRODUCT: Float,
     FOOD_SERVICE: Float,
     HOME_FURNISHING: Float,
     SPORTS: Float,
     OUTDOOR_ACTIVITIES: Float,
     MEDICINE: Float
     )
 
object UserTagTable {
 
 val LOG = LoggerFactory.getLogger(UserOverviewFirst.getClass)
 
 val REP_HOME = s"${AppConfig.HDFS_MASTER}/${AppConfig.HDFS_REP}"
 
 def main(args: Array[String]) {
 
 var startTime = System.currentTimeMillis()
 
 val conf: com.typesafe.config.Config = ConfigFactory.load()
 
 val sc = new SparkContext()
 
 val sqlContext = new SQLContext(sc)
 
 var df1: DataFrame = null
 
 if (args.length == 0) {
  println("请输入: appkey , StartTime : 2016-04-10 ,StartEnd :2016-04-11")
 }
 else {
 
  var appkey = args(0)
 
  var lastdate = args(1)
 
  df1 = loadDataFrame(sqlContext, appkey, "2016-04-10", lastdate)
 
  df1.registerTempTable("suuidTable")
 
  sqlContext.udf.register("taginfo", (a: String) => userTagInfo(a))
  sqlContext.udf.register("intToString", (b: Long) => intToString(b))
  import sqlContext.implicits._
 
  //***重点***:将临时表中的suuid和自定函数中Json数据,放入UserTag中。
 sqlContext.sql(" select distinct(suuid) AS suuid,taginfo(suuid) from suuidTable group by suuid").map { case Row(suuid: String, taginfo: String) =>
  val taginfoObj = JSON.parseObject(taginfo)
  UserTag(suuid.toString,
   taginfoObj.getFloat("man"),
   taginfoObj.getFloat("woman"),
   taginfoObj.getFloat("age10_19"),
   taginfoObj.getFloat("age20_29"),
   taginfoObj.getFloat("age30_39"),
   taginfoObj.getFloat("age40_49"),
   taginfoObj.getFloat("age50_59"),
   taginfoObj.getFloat("game"),
   taginfoObj.getFloat("movie"),
   taginfoObj.getFloat("music"),
   taginfoObj.getFloat("art"),
   taginfoObj.getFloat("politics_news"),
   taginfoObj.getFloat("financial"),
   taginfoObj.getFloat("education_training"),
   taginfoObj.getFloat("health_care"),
   taginfoObj.getFloat("travel"),
   taginfoObj.getFloat("automobile"),
   taginfoObj.getFloat("house_property"),
   taginfoObj.getFloat("clothing_accessories"),
   taginfoObj.getFloat("beauty"),
   taginfoObj.getFloat("IT"),
   taginfoObj.getFloat("baby_Product"),
   taginfoObj.getFloat("food_service"),
   taginfoObj.getFloat("home_furnishing"),
   taginfoObj.getFloat("sports"),
   taginfoObj.getFloat("outdoor_activities"),
   taginfoObj.getFloat("medicine")
  )}.toDF().registerTempTable("resultTable")
 
  val resultDF = sqlContext.sql(s"select '$appkey' AS APPKEY, '$lastdate' AS DATE,SUUID ,MAN,WOMAN,AGE10_19,AGE20_29,AGE30_39 ," +
  "AGE40_49 ,AGE50_59,GAME,MOVIE,MUSIC,ART,POLITICS_NEWS,FINANCIAL,EDUCATION_TRAINING,HEALTH_CARE,TRAVEL,AUTOMOBILE," +
  "HOUSE_PROPERTY,CLOTHING_ACCESSORIES,BEAUTY,IT,BABY_PRODUCT ,FOOD_SERVICE ,HOME_FURNISHING ,SPORTS ,OUTDOOR_ACTIVITIES ," +
  "MEDICINE from resultTable WHERE SUUID IS NOT NULL")
  resultDF.write.mode(SaveMode.Overwrite).options(
  Map("table" -> "USER_TAGS", "zkUrl" -> conf.getString("Hbase.url"))
  ).format("org.apache.phoenix.spark").save()
 
 }
 }
 
 def intToString(suuid: Long): String = {
 suuid.toString()
 }
 
 def userTagInfo(num1: String): String = {
 
 var de = new DecimalFormat("0.00")
 var mannum = de.format(math.random).toFloat
 var man = mannum
 var woman = de.format(1 - mannum).toFloat
 
 var age10_19num = de.format(math.random * 0.2).toFloat
 var age20_29num = de.format(math.random * 0.2).toFloat
 var age30_39num = de.format(math.random * 0.2).toFloat
 var age40_49num = de.format(math.random * 0.2).toFloat
 
 var age10_19 = age10_19num
 var age20_29 = age20_29num
 var age30_39 = age30_39num
 var age40_49 = age40_49num
 var age50_59 = de.format(1 - age10_19num - age20_29num - age30_39num - age40_49num).toFloat
 
 var game = de.format(math.random * 1).toFloat
 var movie = de.format(math.random * 1).toFloat
 var music = de.format(math.random * 1).toFloat
 var art = de.format(math.random * 1).toFloat
 var politics_news = de.format(math.random * 1).toFloat
 
 var financial = de.format(math.random * 1).toFloat
 var education_training = de.format(math.random * 1).toFloat
 var health_care = de.format(math.random * 1).toFloat
 var travel = de.format(math.random * 1).toFloat
 var automobile = de.format(math.random * 1).toFloat
 
 var house_property = de.format(math.random * 1).toFloat
 var clothing_accessories = de.format(math.random * 1).toFloat
 var beauty = de.format(math.random * 1).toFloat
 var IT = de.format(math.random * 1).toFloat
 var baby_Product = de.format(math.random * 1).toFloat
 
 var food_service = de.format(math.random * 1).toFloat
 var home_furnishing = de.format(math.random * 1).toFloat
 var sports = de.format(math.random * 1).toFloat
 var outdoor_activities = de.format(math.random * 1).toFloat
 var medicine = de.format(math.random * 1).toFloat
 
 "{" + "\"man\"" + ":" + man + "," + "\"woman\"" + ":" + woman + "," + "\"age10_19\"" + ":" + age10_19 + "," + "\"age20_29\"" + ":" + age20_29 + "," +
  "\"age30_39\"" + ":" + age30_39 + "," + "\"age40_49\"" + ":" + age40_49 + "," + "\"age50_59\"" + ":" + age50_59 + "," + "\"game\"" + ":" + game + "," +
  "\"movie\"" + ":" + movie + "," + "\"music\"" + ":" + music + "," + "\"art\"" + ":" + art + "," + "\"politics_news\"" + ":" + politics_news + "," +
  "\"financial\"" + ":" + financial + "," + "\"education_training\"" + ":" + education_training + "," + "\"health_care\"" + ":" + health_care + "," +
  "\"travel\"" + ":" + travel + "," + "\"automobile\"" + ":" + automobile + "," + "\"house_property\"" + ":" + house_property + "," + "\"clothing_accessories\"" + ":" + clothing_accessories + "," +
  "\"beauty\"" + ":" + beauty + "," + "\"IT\"" + ":" + IT + "," + "\"baby_Product\"" + ":" + baby_Product + "," + "\"food_service\"" + ":" + food_service + "," +
  "\"home_furnishing\"" + ":" + home_furnishing + "," + "\"sports\"" + ":" + sports + "," + "\"outdoor_activities\"" + ":" + outdoor_activities + "," + "\"medicine\"" + ":" + medicine +
  "}";
 
 }
 
 def loadDataFrame(ctx: SQLContext, appkey: String, startDay: String, endDay: String): DataFrame = {
 val path = s"$REP_HOME/appstatistic"
 ctx.read.parquet(path)
  .filter(s"timestamp is not null and appkey='$appkey' and day>='$startDay' and day<='$endDay'")
 }
 
 
}

以上这篇DataFrame:通过SparkSql将scala类转为DataFrame的方法就是小编分享给大家的全部内容了,希望能给大家一个参考,也希望大家多多支持三水点靠木。

Python 相关文章推荐
用Python制作简单的朴素基数估计器的教程
Apr 01 Python
用Python遍历C盘dll文件的方法
May 06 Python
Python sys.argv用法实例
May 28 Python
Python 爬虫之超链接 url中含有中文出错及解决办法
Aug 03 Python
Ubuntu安装Jupyter Notebook教程
Oct 18 Python
学习python中matplotlib绘图设置坐标轴刻度、文本
Feb 07 Python
Python使用cx_Oracle模块操作Oracle数据库详解
May 07 Python
Python测试模块doctest使用解析
Aug 10 Python
python实现简单井字棋小游戏
Mar 05 Python
Python 实现将大图切片成小图,将小图组合成大图的例子
Mar 14 Python
利用python中的matplotlib打印混淆矩阵实例
Jun 16 Python
详解在Python中使用Torchmoji将文本转换为表情符号
Jul 27 Python
pandas去除重复列的实现方法
Jan 29 #Python
使用Python向C语言的链接库传递数组、结构体、指针类型的数据
Jan 29 #Python
pandas去重复行并分类汇总的实现方法
Jan 29 #Python
spark dataframe 将一列展开,把该列所有值都变成新列的方法
Jan 29 #Python
Python使用ctypes调用C/C++的方法
Jan 29 #Python
dataframe 按条件替换某一列中的值方法
Jan 29 #Python
Numpy之random函数使用学习
Jan 29 #Python
You might like
Phpbean路由转发的php代码
2008/01/10 PHP
PHP隐形一句话后门,和ThinkPHP框架加密码程序(base64_decode)
2011/11/02 PHP
浅析PHP中的字符串编码转换(自动识别原编码)
2013/07/02 PHP
windows服务器中检测PHP SSL是否开启以及开启SSL的方法
2014/04/25 PHP
php中关于socket的系列函数总结
2015/05/18 PHP
PHP生成唯一ID之SnowFlake算法
2016/12/17 PHP
PHP实现在数据库百万条数据中随机获取20条记录的方法
2017/04/19 PHP
php7安装yar扩展的方法详解
2017/08/03 PHP
PHP使用Nginx实现反向代理
2017/09/20 PHP
JS特殊函数(Function()构造函数、函数直接量)区别介绍
2013/05/19 Javascript
文本框回车提交与禁止提交示例
2013/09/27 Javascript
初识Node.js
2014/09/03 Javascript
jQuery的css() 方法使用指南
2015/05/03 Javascript
jquery在ie7下选择器的问题导致append失效的解决方法
2016/01/10 Javascript
js实现统计字符串中特定字符出现个数的方法
2016/08/02 Javascript
Javascript中indexOf()和lastIndexOf应用方法实例
2016/08/24 Javascript
如何使用vuejs实现更好的Form validation?
2017/04/07 Javascript
深入理解JavaScript创建对象的多种方式以及优缺点
2017/06/01 Javascript
promise处理多个相互依赖的异步请求(实例讲解)
2017/08/03 Javascript
[00:36]我的中国心——Serenity vs Fnatic
2018/08/21 DOTA
Python使用urllib2获取网络资源实例讲解
2013/12/02 Python
浅谈python中的实例方法、类方法和静态方法
2017/02/17 Python
对Python获取屏幕截图的4种方法详解
2019/08/27 Python
Win下PyInstaller 安装和使用教程
2019/12/25 Python
python解析xml文件方式(解析、更新、写入)
2020/03/05 Python
使用pyplot.matshow()函数添加绘图标题
2020/06/16 Python
Python生成器generator原理及用法解析
2020/07/20 Python
PyQt5的QWebEngineView使用示例
2020/10/20 Python
使用Python下载抖音各大V视频的思路详解
2021/02/06 Python
找到您丢失的钥匙、钱包和手机:Tile
2017/05/19 全球购物
Radley英国官网:英国莱德利小狗包
2019/03/21 全球购物
Monica Vinader官网:英国轻奢珠宝品牌
2020/02/05 全球购物
Java基础知识面试题
2014/03/25 面试题
简易离婚协议书范本
2014/10/24 职场文书
委托公证书样本
2015/01/23 职场文书
python库Tsmoothie模块数据平滑化异常点抓取
2022/06/10 Python