Commit d4fe6410 by WangJinfeng

init id_mapping

parent 2b5304c6
......@@ -94,15 +94,14 @@ class IDMappingGraphx extends CommonSparkJob with Serializable {
processVertex(date, row, idSet, idMainSet)
}).flatMap(l => l)
val maxGraph = vertex.filter(kv => {
// 非主ID生成OneID
val multiOneIDRDD = vertex.filter(kv => {
!idMainSet.contains(kv._2._3)
}).combineByKey(
(v: (String, String, String)) => Set(v),
(c: Set[(String, String, String)], v: (String, String, String)) => c ++ Seq(v),
(c1: Set[(String, String, String)], c2: Set[(String, String, String)]) => c1 ++ c2
)
val multiOneIDRDD = maxGraph.map(rs => {
).map(rs => {
platform match {
case "ios" =>
updateOneID(rs, Constant.iosMainIDSet)
......@@ -111,12 +110,12 @@ class IDMappingGraphx extends CommonSparkJob with Serializable {
}
}).flatMap(l => l)
// 主ID生成OneID
val singleOneIDRDD = vertex.filter(kv => {
idMainSet.contains(kv._2._3)
// kv._2.size == 1
}).map(kv => {
val oneID = new JSONObject()
oneID.put(kv._2._1, kv._2._2)
oneID.put(kv._2._1, MobvistaConstant.String2JSONObject(kv._2._2))
(kv._1, oneID.toJSONString, kv._2._3)
})
/*
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment