Skip to content

M
mobvista-dmp
Commit 84176322 by fan.jiang
Options

add dealid to dsp_req business

parent 592aad84
Showing with 123 additions and 167 deletions
src/main/scala/mobvista/dmp/datasource/dsp/DspOrgEtlDailys.scala
......@@ -341,99 +341,56 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
val dealeridArray: Array[String] = dealerid.split(",")
if ("oppocn".equals(exchanges)) {
if("android".equalsIgnoreCase(platform)) {
if (dealeridArray.contains("2532") || dealeridArray.contains("2533")) {
value.packageName = "com.taobao.taobao"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = "com.taobao.taobao_oppo"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
} else {
if ( !(dealeridArray.contains("2532") || dealeridArray.contains("2533")) ){
// 2020.11.26 需求 若adx（exchanges字段）为oppo且dealid不等于2532或2533，则伪包名为com.taobao.taobao_notinstall_oppo
value.packageName = "com.taobao.taobao_notinstall_oppo"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
// 添加com.UCMobile 对应伪包名小写com.ucmobile_oppo
if (dealeridArray.contains("2728")) {
value.packageName = "com.UCMobile"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = "com.ucmobile_oppo"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
val mapData_oppoziyou= Map("4059" -> "com.ss.android.ugc.aweme",
"4140" -> "com.xunmeng.pinduoduo",
"4183" -> "me.ele")
for (item <- mapData_oppoziyou) {
if (dealeridArray.contains(item._1)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = item._2 + "_oppoziyou"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
val mapData_oppolianmeng= Map("4053" -> "com.ss.android.ugc.aweme",
"4143" -> "com.xunmeng.pinduoduo",
"4182" -> "me.ele")
for (item <- mapData_oppolianmeng) {
val mapData_oppo= Map(
"2532" -> Array("com.taobao.taobao","com.taobao.taobao_oppo"),
"2533" -> Array("com.taobao.taobao","com.taobao.taobao_oppo"),
"4053" -> Array("com.ss.android.ugc.aweme","com.ss.android.ugc.aweme_oppolianmeng"),
"4143" -> Array("com.xunmeng.pinduoduo","com.xunmeng.pinduoduo_oppolianmeng"),
"2906" -> Array("com.youku.phone","com.youku.phone_oppolianmeng"),
"4182" -> Array("me.ele","me.ele_oppolianmeng"),
"4059" -> Array("com.ss.android.ugc.aweme","com.ss.android.ugc.aweme_oppoziyou"),
"4140" -> Array("com.xunmeng.pinduoduo","com.xunmeng.pinduoduo_oppoziyou"),
"4183" -> Array("me.ele","me.ele_oppoziyou"),
"2728" -> Array("com.UCMobile","com.ucmobile_oppo"), // 添加com.UCMobile 对应伪包名小写com.ucmobile_oppo
"4060" -> Array("com.ss.android.ugc.aweme_oppoziyou_notinstall"),
"4061" -> Array("com.ss.android.ugc.aweme_oppoziyou_hist_notinstall"),
"4054" -> Array("com.ss.android.ugc.aweme_oppolianmeng_hist1year_notinstall"),
"4055" -> Array("com.ss.android.ugc.aweme_oppolianmeng_histhalfyear_notinstall"),
"4141" -> Array("com.xunmeng.pinduoduo_oppoziyou_notinstall"),
"4142" -> Array("com.xunmeng.pinduoduo_oppoziyou_hist_notinstall"),
"4144" -> Array("com.xunmeng.pinduoduo_oppolianmeng_hist1year_notinstall"),
"4145" -> Array("com.xunmeng.pinduoduo_oppolianmeng_histhalfyear_notinstall"),
"4184" -> Array("me.ele_oppoziyou_hist_notinstall"),
"4185" -> Array("me.ele_oppoziyou_notinstall"),
"4180" -> Array("me.ele_oppolianmeng_notinstall"),
"2840" -> Array("com.youku.phone_oppocn_notinstall"),
"4181" -> Array("me.ele_oppolianmeng_hist_notinstall"),
"2774" -> Array("com.smile.gifmaker_notinstall_oppo"), //2020.11.11添加快手、京东入库
"2773" -> Array("com.jingdong.app.mallr_notinstall_oppo"),
"2716" -> Array("com.jingdong.app.mall","com.jingdong.app.mall_oppo"),
"2717" -> Array("com.eg.android.AlipayGphone","com.eg.android.AlipayGphone_oppo"), //补充说明： com.eg.android.AlipayGphone_oppo伪包名大写，但已经入库，修改困难。以后入库伪包名统一定为小写，类似上面的 com.ucmobile_oppo）
"2718" -> Array("com.qiyi.video","com.qiyi.video_oppo"),
"2783" -> Array("com.taobao.idlefish","com.taobao.idlefish_oppo"),
"2840" -> Array("com.youku.phone_notinstall","com.youku.phone_notinstall_oppo"),
"2889" -> Array("com.sankuai.meituan","com.sankuai.meituan_oppo"),
"2890" -> Array("com.meituan.itakeaway","com.meituan.itakeaway_oppo"),
"2904" -> Array("com.smile.gifmaker","com.smile.gifmaker_oppo"),
"2905" -> Array("com.kuaishou.nebula","com.kuaishou.nebula_oppo"),
"2906" -> Array("com.youku.phone","com.youku.phone_oppo"),
"3160" -> Array("com.tencent.news","com.tencent.news_oppo")
)
for (item <- mapData_oppo) {
if (dealeridArray.contains(item._1)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = item._2 + "_oppolianmeng"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
val mapData_oppocn: Map[Int, String] = Map(4060 -> "com.ss.android.ugc.aweme_oppoziyou_notinstall",
4061 -> "com.ss.android.ugc.aweme_oppoziyou_hist_notinstall",
4054 -> "com.ss.android.ugc.aweme_oppolianmeng_hist1year_notinstall",
4055 -> "com.ss.android.ugc.aweme_oppolianmeng_histhalfyear_notinstall",
4141 -> "com.xunmeng.pinduoduo_oppoziyou_notinstall",
4142 -> "com.xunmeng.pinduoduo_oppoziyou_hist_notinstall",
4144 -> "com.xunmeng.pinduoduo_oppolianmeng_hist1year_notinstall",
4145 -> "com.xunmeng.pinduoduo_oppolianmeng_histhalfyear_notinstall",
4184 -> "me.ele_oppoziyou_hist_notinstall",
4185 -> "me.ele_oppoziyou_notinstall",
4180 -> "me.ele_oppolianmeng_notinstall",
4181 -> "me.ele_oppolianmeng_hist_notinstall")
for (item <- mapData_oppocn) {
if (dealeridArray.contains(item._1.toString)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
// 添加 oppo 在请求中开始传支付宝，京东，爱奇艺这几个包名的安装信息，并入库。 （2020-10-28 17：17 补充说明： com.eg.android.AlipayGphone_oppo伪包名大写，但已经入库，修改困难。以后入库伪包名统一定为小写，类似上面的 com.ucmobile_oppo）
// 2020.11.11 添加 2783对应的咸鱼的 com.taobao.idlefish 和 com.taobao.idlefish_oppo 入库
// 2020.11.26 添加 2840对应的 com.youku.phone_notinstall 和 com.youku.phone_notinstall_oppo 入库
// 2020.12.24 添加 2889对应的 com.sankuai.meituan和 com.sankuai.meituan_oppo 入库
// 2020.12.24 添加 2890对应的 com.meituan.itakeaway和 com.meituan.itakeaway_oppo 入库
// 2021.01.06添加 快手、快手极速版、优酷
// 2021.04.22 添加adx（exchanges字段）为oppo且dealid=3160，则伪包名为com.tencent.news_oppo和com.tencent.news
var mapData: Map[Int, String] = Map(2716 -> "com.jingdong.app.mall",
2717 -> "com.eg.android.AlipayGphone",
2718 -> "com.qiyi.video",
2783 -> "com.taobao.idlefish",
2840 -> "com.youku.phone_notinstall",
2889 -> "com.sankuai.meituan",
2890 -> "com.meituan.itakeaway",
2904 -> "com.smile.gifmaker",
2905 -> "com.kuaishou.nebula",
2906 -> "com.youku.phone",
3160 -> "com.tencent.news")
for (item <- mapData) {
if (dealeridArray.contains(item._1.toString)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = item._2 + "_oppo"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
//2020.11.11添加快手、京东入库
mapData = Map(2774 -> "com.smile.gifmaker_notinstall_oppo",
2773 -> "com.jingdong.app.mallr_notinstall_oppo")
for (item <- mapData) {
if (dealeridArray.contains(item._1.toString)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
for( packageName <- item._2 ){
value.packageName = packageName
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
}
......@@ -450,48 +407,46 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
// wiki https://confluence.mobvista.com/pages/viewpage.action?pageId=47976499
//2021.04.22 添加adx（exchanges字段）为bes且dealid=100188，则伪包名为com.tencent.news_bes和com.tencent.news
//2021.06.17 添加adx（exchanges字段）为bes且dealid=100310，则伪包名为com.taobao.litetao_bes和com.taobao.litetao
val mapData_bes = Map("100193" -> "com.taobao.taobao",
"100189" -> "com.eg.android.AlipayGphone",
"100191" -> "com.jingdong.app.mall",
"100187" -> "com.UCMobile",
"100194" -> "com.taobao.idlefish",
"100195" -> "com.qiyi.video",
"100196" -> "com.smile.gifmaker",
"100197" -> "id387682726",
"100188" -> "com.tencent.news",
"100310" -> "com.taobao.litetao",
"100203" -> "com.ss.android.ugc.aweme",
"00251" -> "com.xunmeng.pinduoduo",
"100304" -> "me.ele")
val mapData_ios_bes = Map("100197" -> "id387682726",
"100345" -> "id333206289",
"100344" -> "id1340376323",
"100353" -> "id1044283059",
"100368" -> "id507161324")
val mapData_bes = Map("100193" -> Array("com.taobao.taobao","com.taobao.taobao_bes"),
"100189" -> Array("com.eg.android.AlipayGphone","com.eg.android.AlipayGphone_bes"),
"100191" -> Array("com.jingdong.app.mall","com.jingdong.app.mall_bes"),
"100187" -> Array("com.UCMobile","com.UCMobile_bes"),
"100194" -> Array("com.taobao.idlefish","com.taobao.idlefish_bes"),
"100195" -> Array("com.qiyi.video","com.qiyi.video_bes"),
"100196" -> Array("com.smile.gifmaker","com.smile.gifmaker_bes"),
"100188" -> Array("com.tencent.news","com.tencent.news_bes"),
"100310" -> Array("com.taobao.litetao","com.taobao.litetao_bes"),
"100203" -> Array("com.ss.android.ugc.aweme","com.ss.android.ugc.aweme_bes"),
"00251" -> Array("com.xunmeng.pinduoduo","com.xunmeng.pinduoduo_bes"),
"100400" -> Array("com.youku.phone","com.youku.phone_bes"),
"100219" -> Array("com.youku.phone_bes_notinstall"),
"100304" -> Array("me.ele","me.ele_bes") )
val mapData_ios_bes = Map("100197" -> Array("id387682726","id3876827262021090301"),
"100345" -> Array("id333206289","id3332062892021090301"),
"100344" -> Array("id1340376323","id13403763232021090301"),
"100353" -> Array("id1044283059","id10442830592021090301"),
"100220" -> Array("id3361414752022010401"),
"100368" -> Array("id507161324","id5071613242021091701") )
if ("bes".equals(exchanges)) {
if("android".equalsIgnoreCase(platform)) {
for (item <- mapData_bes) {
if (dealeridArray.contains(item._1)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = item._2 + "_bes"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
for( packageName <- item._2 ){
value.packageName = packageName
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
}
if("ios".equalsIgnoreCase(platform)) {
for (item <- mapData_ios_bes) {
if (dealeridArray.contains(item._1)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
if("100368".equalsIgnoreCase(item._1)){
value.packageName = item._2 + "2021091701"
}else{
value.packageName = item._2 + "2021090301"
for( packageName <- item._2 ){
value.packageName = packageName
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
......@@ -519,67 +474,68 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
}
}
val mapData_iqiyi = Map(5260 -> "com.taobao.taobao",
1301 -> "com.UCMobile",
10949 -> "com.eg.android.AlipayGphone",
6242 -> "com.taobao.idlefish",
3996 -> "com.sankuai.meituan",
3334 -> "com.ss.android.ugc.aweme",
112799 -> "com.ss.android.ugc.aweme",
7156 -> "com.tencent.news",
1649 -> "me.ele")
val mapData_iqiyi = Map(5260 -> Array("com.taobao.taobao","com.taobao.taobao_iqiyi"),
1301 -> Array("com.UCMobile","com.UCMobile_iqiyi"),
10949 -> Array("com.eg.android.AlipayGphone","com.eg.android.AlipayGphone_iqiyi"),
6242 -> Array("com.taobao.idlefish","com.taobao.idlefish_iqiyi"),
3996 -> Array("com.sankuai.meituan","com.sankuai.meituan_iqiyi"),
3334 -> Array("com.ss.android.ugc.aweme","com.ss.android.ugc.aweme_iqiyi"),
112799 -> Array("com.ss.android.ugc.aweme","com.ss.android.ugc.aweme_iqiyi"),
7156 -> Array("com.tencent.news","com.tencent.news_iqiyi"),
112644 -> Array("com.taobao.litetao","com.taobao.litetao_iqiyi"),
1649 -> Array("me.ele","me.ele_iqiyi") )
val mapData_ios_iqiyi = Map(
3334 -> "id1142110895",
112799 -> "id1142110895",
1649 -> "id507161324")
3334 -> Array("id1142110895","id11421108952021090302"),
112799 -> Array("id1142110895","id11421108952021090302"),
112644 -> Array("1340376323","134037632320210617"),
1649 -> Array("id507161324","id5071613242021091702"))
if("iqiyi".equals(exchanges)){
if("android".equalsIgnoreCase(platform)) {
for (item <- mapData_iqiyi) {
if (dealeridArray.contains(item._1.toString)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = item._2 + "_iqiyi"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
for (item <- mapData_iqiyi) {
if (dealeridArray.contains(item._1.toString)) {
for( packageName <- item._2 ){
value.packageName = packageName
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
}
if("ios".equalsIgnoreCase(platform)) {
for (item <- mapData_ios_iqiyi) {
if (dealeridArray.contains(item._1.toString)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
if("1649".equalsIgnoreCase(item._1.toString)){
value.packageName = item._2 + "2021091702"
}else{
value.packageName = item._2 + "2021090302"
}
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
//2021.06.17 若adx（exchanges字段）为iqiyi且dealid=112644(安装) 且os = 'android'，则伪包名为com.taobao.litetao_iqiyi和com.taobao.litetao 若adx（exchanges字段）为iqiyi且dealid=112644（安装）且os='ios'，则安装包名为134037632320210617和1340376323
if (dealeridArray.contains("112644")) {
if("android".equalsIgnoreCase(platform)){
value.packageName = "com.taobao.litetao"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, "", deviceType, "android", value)
value.packageName = "com.taobao.litetao_iqiyi"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, "", deviceType, "android", value)
}else{
value.packageName = "1340376323"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, "", "", "", "", "", "", idfv, deviceType, "ios", value)
value.packageName = "134037632320210617"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, "", "", "", "", "", "", idfv, deviceType, "ios", value)
}
for (item <- mapData_ios_iqiyi) {
if (dealeridArray.contains(item._1.toString)) {
for( packageName <- item._2 ){
value.packageName = packageName
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
}
}
val mapData_vivo = Map("ba6e4f5323603561086a985b6fc5ffb4" -> Array("me.ele","me.ele_vivo"),
"d326b636626b0d927be609d317b88cae" -> Array("com.youku.phone","com.youku.phone_vivo"))
val mapData_ios_vivo = Map(
"d326b636626b0d927be609d317b88cae" -> Array("id336141475","id3361414752022010402") )
if ("vivo".equals(exchanges)) {
if("android".equalsIgnoreCase(platform)) {
if (dealeridArray.contains("ba6e4f5323603561086a985b6fc5ffb4")) {
value.packageName = "me.ele"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = "me.ele_vivo"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
for (item <- mapData_vivo) {
if (dealeridArray.contains(item._1)) {
for( packageName <- item._2 ){
value.packageName = packageName
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
}
if("ios".equalsIgnoreCase(platform)) {
for (item <- mapData_ios_vivo) {
if (dealeridArray.contains(item._1)) {
for( packageName <- item._2 ){
value.packageName = packageName
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment