DataFrame:通过SparkSql将scala类转为DataFrame的方法


Posted in Python onJanuary 29, 2019

如下所示:

import java.text.DecimalFormat
import com.alibaba.fastjson.JSON
import com.donews.data.AppConfig
import com.typesafe.config.ConfigFactory
import org.apache.spark.sql.types.{StructField, StructType}
import org.apache.spark.sql.{Row, SaveMode, DataFrame, SQLContext}
import org.apache.spark.{SparkConf, SparkContext}
import org.slf4j.LoggerFactory
 
/**
 * Created by silentwolf on 2016/6/3.
 */
 
case class UserTag(SUUID: String,
     MAN: Float,
     WOMAN: Float,
     AGE10_19: Float,
     AGE20_29: Float,
     AGE30_39: Float,
     AGE40_49: Float,
     AGE50_59: Float,
     GAME: Float,
     MOVIE: Float,
     MUSIC: Float,
     ART: Float,
     POLITICS_NEWS: Float,
     FINANCIAL: Float,
     EDUCATION_TRAINING: Float,
     HEALTH_CARE: Float,
     TRAVEL: Float,
     AUTOMOBILE: Float,
     HOUSE_PROPERTY: Float,
     CLOTHING_ACCESSORIES: Float,
     BEAUTY: Float,
     IT: Float,
     BABY_PRODUCT: Float,
     FOOD_SERVICE: Float,
     HOME_FURNISHING: Float,
     SPORTS: Float,
     OUTDOOR_ACTIVITIES: Float,
     MEDICINE: Float
     )
 
object UserTagTable {
 
 val LOG = LoggerFactory.getLogger(UserOverviewFirst.getClass)
 
 val REP_HOME = s"${AppConfig.HDFS_MASTER}/${AppConfig.HDFS_REP}"
 
 def main(args: Array[String]) {
 
 var startTime = System.currentTimeMillis()
 
 val conf: com.typesafe.config.Config = ConfigFactory.load()
 
 val sc = new SparkContext()
 
 val sqlContext = new SQLContext(sc)
 
 var df1: DataFrame = null
 
 if (args.length == 0) {
  println("请输入: appkey , StartTime : 2016-04-10 ,StartEnd :2016-04-11")
 }
 else {
 
  var appkey = args(0)
 
  var lastdate = args(1)
 
  df1 = loadDataFrame(sqlContext, appkey, "2016-04-10", lastdate)
 
  df1.registerTempTable("suuidTable")
 
  sqlContext.udf.register("taginfo", (a: String) => userTagInfo(a))
  sqlContext.udf.register("intToString", (b: Long) => intToString(b))
  import sqlContext.implicits._
 
  //***重点***:将临时表中的suuid和自定函数中Json数据,放入UserTag中。
 sqlContext.sql(" select distinct(suuid) AS suuid,taginfo(suuid) from suuidTable group by suuid").map { case Row(suuid: String, taginfo: String) =>
  val taginfoObj = JSON.parseObject(taginfo)
  UserTag(suuid.toString,
   taginfoObj.getFloat("man"),
   taginfoObj.getFloat("woman"),
   taginfoObj.getFloat("age10_19"),
   taginfoObj.getFloat("age20_29"),
   taginfoObj.getFloat("age30_39"),
   taginfoObj.getFloat("age40_49"),
   taginfoObj.getFloat("age50_59"),
   taginfoObj.getFloat("game"),
   taginfoObj.getFloat("movie"),
   taginfoObj.getFloat("music"),
   taginfoObj.getFloat("art"),
   taginfoObj.getFloat("politics_news"),
   taginfoObj.getFloat("financial"),
   taginfoObj.getFloat("education_training"),
   taginfoObj.getFloat("health_care"),
   taginfoObj.getFloat("travel"),
   taginfoObj.getFloat("automobile"),
   taginfoObj.getFloat("house_property"),
   taginfoObj.getFloat("clothing_accessories"),
   taginfoObj.getFloat("beauty"),
   taginfoObj.getFloat("IT"),
   taginfoObj.getFloat("baby_Product"),
   taginfoObj.getFloat("food_service"),
   taginfoObj.getFloat("home_furnishing"),
   taginfoObj.getFloat("sports"),
   taginfoObj.getFloat("outdoor_activities"),
   taginfoObj.getFloat("medicine")
  )}.toDF().registerTempTable("resultTable")
 
  val resultDF = sqlContext.sql(s"select '$appkey' AS APPKEY, '$lastdate' AS DATE,SUUID ,MAN,WOMAN,AGE10_19,AGE20_29,AGE30_39 ," +
  "AGE40_49 ,AGE50_59,GAME,MOVIE,MUSIC,ART,POLITICS_NEWS,FINANCIAL,EDUCATION_TRAINING,HEALTH_CARE,TRAVEL,AUTOMOBILE," +
  "HOUSE_PROPERTY,CLOTHING_ACCESSORIES,BEAUTY,IT,BABY_PRODUCT ,FOOD_SERVICE ,HOME_FURNISHING ,SPORTS ,OUTDOOR_ACTIVITIES ," +
  "MEDICINE from resultTable WHERE SUUID IS NOT NULL")
  resultDF.write.mode(SaveMode.Overwrite).options(
  Map("table" -> "USER_TAGS", "zkUrl" -> conf.getString("Hbase.url"))
  ).format("org.apache.phoenix.spark").save()
 
 }
 }
 
 def intToString(suuid: Long): String = {
 suuid.toString()
 }
 
 def userTagInfo(num1: String): String = {
 
 var de = new DecimalFormat("0.00")
 var mannum = de.format(math.random).toFloat
 var man = mannum
 var woman = de.format(1 - mannum).toFloat
 
 var age10_19num = de.format(math.random * 0.2).toFloat
 var age20_29num = de.format(math.random * 0.2).toFloat
 var age30_39num = de.format(math.random * 0.2).toFloat
 var age40_49num = de.format(math.random * 0.2).toFloat
 
 var age10_19 = age10_19num
 var age20_29 = age20_29num
 var age30_39 = age30_39num
 var age40_49 = age40_49num
 var age50_59 = de.format(1 - age10_19num - age20_29num - age30_39num - age40_49num).toFloat
 
 var game = de.format(math.random * 1).toFloat
 var movie = de.format(math.random * 1).toFloat
 var music = de.format(math.random * 1).toFloat
 var art = de.format(math.random * 1).toFloat
 var politics_news = de.format(math.random * 1).toFloat
 
 var financial = de.format(math.random * 1).toFloat
 var education_training = de.format(math.random * 1).toFloat
 var health_care = de.format(math.random * 1).toFloat
 var travel = de.format(math.random * 1).toFloat
 var automobile = de.format(math.random * 1).toFloat
 
 var house_property = de.format(math.random * 1).toFloat
 var clothing_accessories = de.format(math.random * 1).toFloat
 var beauty = de.format(math.random * 1).toFloat
 var IT = de.format(math.random * 1).toFloat
 var baby_Product = de.format(math.random * 1).toFloat
 
 var food_service = de.format(math.random * 1).toFloat
 var home_furnishing = de.format(math.random * 1).toFloat
 var sports = de.format(math.random * 1).toFloat
 var outdoor_activities = de.format(math.random * 1).toFloat
 var medicine = de.format(math.random * 1).toFloat
 
 "{" + "\"man\"" + ":" + man + "," + "\"woman\"" + ":" + woman + "," + "\"age10_19\"" + ":" + age10_19 + "," + "\"age20_29\"" + ":" + age20_29 + "," +
  "\"age30_39\"" + ":" + age30_39 + "," + "\"age40_49\"" + ":" + age40_49 + "," + "\"age50_59\"" + ":" + age50_59 + "," + "\"game\"" + ":" + game + "," +
  "\"movie\"" + ":" + movie + "," + "\"music\"" + ":" + music + "," + "\"art\"" + ":" + art + "," + "\"politics_news\"" + ":" + politics_news + "," +
  "\"financial\"" + ":" + financial + "," + "\"education_training\"" + ":" + education_training + "," + "\"health_care\"" + ":" + health_care + "," +
  "\"travel\"" + ":" + travel + "," + "\"automobile\"" + ":" + automobile + "," + "\"house_property\"" + ":" + house_property + "," + "\"clothing_accessories\"" + ":" + clothing_accessories + "," +
  "\"beauty\"" + ":" + beauty + "," + "\"IT\"" + ":" + IT + "," + "\"baby_Product\"" + ":" + baby_Product + "," + "\"food_service\"" + ":" + food_service + "," +
  "\"home_furnishing\"" + ":" + home_furnishing + "," + "\"sports\"" + ":" + sports + "," + "\"outdoor_activities\"" + ":" + outdoor_activities + "," + "\"medicine\"" + ":" + medicine +
  "}";
 
 }
 
 def loadDataFrame(ctx: SQLContext, appkey: String, startDay: String, endDay: String): DataFrame = {
 val path = s"$REP_HOME/appstatistic"
 ctx.read.parquet(path)
  .filter(s"timestamp is not null and appkey='$appkey' and day>='$startDay' and day<='$endDay'")
 }
 
 
}

以上这篇DataFrame:通过SparkSql将scala类转为DataFrame的方法就是小编分享给大家的全部内容了,希望能给大家一个参考,也希望大家多多支持三水点靠木。

Python 相关文章推荐
python在命令行下使用google翻译(带语音)
Jan 16 Python
python基于queue和threading实现多线程下载实例
Oct 08 Python
Python字符串、元组、列表、字典互相转换的方法
Jan 23 Python
Python简单计算文件MD5值的方法示例
Apr 11 Python
python日期相关操作实例小结
Jun 24 Python
利用python numpy+matplotlib绘制股票k线图的方法
Jun 26 Python
Python如何使用OS模块调用cmd
Feb 27 Python
浅谈python3 构造函数和析构函数
Mar 12 Python
Python3爬虫关于代理池的维护详解
Jul 30 Python
python3获取控制台输入的数据的具体实例
Aug 16 Python
如何用Python 实现全连接神经网络(Multi-layer Perceptron)
Oct 15 Python
python 下载文件的几种方法汇总
Jan 06 Python
pandas去除重复列的实现方法
Jan 29 #Python
使用Python向C语言的链接库传递数组、结构体、指针类型的数据
Jan 29 #Python
pandas去重复行并分类汇总的实现方法
Jan 29 #Python
spark dataframe 将一列展开,把该列所有值都变成新列的方法
Jan 29 #Python
Python使用ctypes调用C/C++的方法
Jan 29 #Python
dataframe 按条件替换某一列中的值方法
Jan 29 #Python
Numpy之random函数使用学习
Jan 29 #Python
You might like
功能齐全的PHP发送邮件类代码附详细说明
2008/07/10 PHP
基于PHP CURL用法的深入分析
2013/06/09 PHP
PHP图形操作之Jpgraph学习笔记
2015/12/25 PHP
PHP调用存储过程返回值不一致问题的解决方法分析
2016/04/26 PHP
使用JavaScript创建新样式表和新样式规则
2016/06/14 PHP
javascript web对话框与弹出窗口
2009/02/22 Javascript
Jquery选中或取消radio示例
2013/09/29 Javascript
node.js中的fs.truncateSync方法使用说明
2014/12/15 Javascript
javascript面向对象快速入门实例
2015/01/13 Javascript
jquery实现的美女拼图游戏实例
2015/05/04 Javascript
Bootstrap每天必学之简单入门
2015/11/19 Javascript
浅析BootStrap模态框的使用(经典)
2016/04/29 Javascript
浅谈javascript:两种注释,声明变量,定义函数
2016/10/05 Javascript
jquery中用jsonp实现搜索框功能
2016/10/18 Javascript
Vue.js系列之项目结构说明(2)
2017/01/03 Javascript
Jquery根据浏览器窗口改变调整大小的方法
2017/02/07 Javascript
js随机生成一个验证码
2017/06/01 Javascript
jQuery+Ajax实现用户名重名实时检测
2017/06/01 jQuery
Python实现视频下载功能
2017/03/14 Python
pyqt远程批量执行Linux命令程序的方法
2019/02/14 Python
wxPython实现分隔窗口
2019/11/19 Python
python抓取多种类型的页面方法实例
2019/11/20 Python
tensorflow之变量初始化(tf.Variable)使用详解
2020/02/06 Python
python字符串常用方法及文件简单读写的操作方法
2020/03/04 Python
django admin 添加自定义链接方式
2020/03/11 Python
python 利用toapi库自动生成api
2020/10/19 Python
python3中布局背景颜色代码分析
2020/12/01 Python
css3实现小箭头各种图形效果
2020/07/08 HTML / CSS
日本PLST在线商店:日本时尚杂志刊载的人气服装
2016/12/10 全球购物
法国二手手袋、手表和奢侈珠宝购物网站:Collector Square
2018/07/05 全球购物
SportsDirect.com马来西亚:英国第一体育零售商
2018/11/21 全球购物
为什么要做架构设计
2015/07/08 面试题
社会实践感言
2014/01/25 职场文书
干部选拔任用方案
2014/05/26 职场文书
党的群众路线教育实践活动党员个人整改措施
2014/10/27 职场文书
大学生团员个人总结
2015/02/14 职场文书