Skip to content

M
mobvista-dmp
Commit b4a41d2c by WangJinfeng
Options

init id_mapping

parent ff5d6e34
Showing with 3 additions and 3 deletions
src/main/scala/mobvista/dmp/datasource/id_mapping/IDMappingGraphx.scala
...@@ -91,7 +91,7 @@ class IDMappingGraphx extends CommonSparkJob with Serializable { ...@@ -91,7 +91,7 @@ class IDMappingGraphx extends CommonSparkJob with Serializable {
}), schema = schame) }), schema = schame)
val vertex = todayDF.rdd.map(row => { val vertex = todayDF.rdd.map(row => {
processVertex(date, row, idSet.toArray, idMainSet) processVertex(date, row, idSet, idMainSet)
}).flatMap(l => l) }).flatMap(l => l)
val maxGraph = vertex.combineByKey( val maxGraph = vertex.combineByKey(
...@@ -265,8 +265,8 @@ class IDMappingGraphx extends CommonSparkJob with Serializable { ...@@ -265,8 +265,8 @@ class IDMappingGraphx extends CommonSparkJob with Serializable {
val array = new ArrayBuffer[(String, JSONObject, String)]() val array = new ArrayBuffer[(String, JSONObject, String)]()
val iters = kv._2 val iters = kv._2
val oneID = new JSONObject() val oneID = new JSONObject()
iters.foreach(kv => { iters.foreach(ir => {
oneID.put(kv._1, kv._2) oneID.put(ir._1, ir._2)
}) })
iters.filter(tp => { iters.filter(tp => {
!mainIDSet.contains(tp._2.getString("id_type")) !mainIDSet.contains(tp._2.getString("id_type"))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment