Commit 69359cfa by WangJinfeng

init id_mapping

parent 408f3263
......@@ -169,8 +169,6 @@ class IDMappingGraphx extends CommonSparkJob with Serializable {
.option("orc.compress", "zlib")
.orc(outPutPath)
FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(resultOutPutPath), true)
val resultOneID = midMergeOneIDRDD.mapPartitions(rs => {
rs.map(r => {
val device_id = r.device_id
......@@ -198,6 +196,9 @@ class IDMappingGraphx extends CommonSparkJob with Serializable {
Result(device_id = device_id, device_type = device_type, one_id = json.toJSONString)
})
})
FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(resultOutPutPath), true)
resultOneID
.toDF
.repartition(coalesce)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment