Commit 69359cfa by WangJinfeng

init id_mapping

parent 408f3263
...@@ -169,8 +169,6 @@ class IDMappingGraphx extends CommonSparkJob with Serializable { ...@@ -169,8 +169,6 @@ class IDMappingGraphx extends CommonSparkJob with Serializable {
.option("orc.compress", "zlib") .option("orc.compress", "zlib")
.orc(outPutPath) .orc(outPutPath)
FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(resultOutPutPath), true)
val resultOneID = midMergeOneIDRDD.mapPartitions(rs => { val resultOneID = midMergeOneIDRDD.mapPartitions(rs => {
rs.map(r => { rs.map(r => {
val device_id = r.device_id val device_id = r.device_id
...@@ -198,6 +196,9 @@ class IDMappingGraphx extends CommonSparkJob with Serializable { ...@@ -198,6 +196,9 @@ class IDMappingGraphx extends CommonSparkJob with Serializable {
Result(device_id = device_id, device_type = device_type, one_id = json.toJSONString) Result(device_id = device_id, device_type = device_type, one_id = json.toJSONString)
}) })
}) })
FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(resultOutPutPath), true)
resultOneID resultOneID
.toDF .toDF
.repartition(coalesce) .repartition(coalesce)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment