dsp_req分区adx dealid入库需求

src/main/scala/mobvista/dmp/datasource/dsp/DspOrgEtlDailys.scala

@@ -340,62 +340,47 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
     val dealerid = row.getAs[String]("dealerid")
     val dealeridArray: Array[String] = dealerid.split(",")
     if ("oppocn".equals(exchanges)) {
+      if("android".equalsIgnoreCase(platform)) {
         if (dealeridArray.contains("2532") || dealeridArray.contains("2533")) {
-        /*
-        arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
-          "com.taobao.taobao", androidId, time, segment, region)
-        arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
-          "com.taobao.taobao_oppo", androidId, time, segment, region)
-        */
           value.packageName = "com.taobao.taobao"
           arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
           value.packageName = "com.taobao.taobao_oppo"
           arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
         } else {
           // 2020.11.26 需求 若adx（exchanges字段）为oppo且dealid不等于2532或2533，则伪包名为com.taobao.taobao_notinstall_oppo
-        /*
-        arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
-          "com.taobao.taobao_notinstall_oppo", androidId, time, segment, region)
-        */
           value.packageName = "com.taobao.taobao_notinstall_oppo"
           arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
         }
         //  添加com.UCMobile 对应伪包名小写com.ucmobile_oppo
         if (dealeridArray.contains("2728")) {
-        /*
-        arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
-          "com.UCMobile", androidId, time, segment, region)
-        arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
-          "com.ucmobile_oppo", androidId, time, segment, region)
-        */
           value.packageName = "com.UCMobile"
           arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
           value.packageName = "com.ucmobile_oppo"
           arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
         }
-      if (dealeridArray.contains("4059")) {
-        value.packageName = "com.ss.android.ugc.aweme"
+
+        val mapData_oppoziyou= Map("4059" -> "com.ss.android.ugc.aweme",
+          "4140" -> "com.xunmeng.pinduoduo",
+          "4183" -> "me.ele")
+        for (item <- mapData_oppoziyou) {
+          if (dealeridArray.contains(item._1)) {
+            value.packageName = item._2
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
-        value.packageName = "com.ss.android.ugc.aweme_oppoziyou"
+            value.packageName = item._2 + "_oppoziyou"
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
           }
-      if (dealeridArray.contains("4053")) {
-        value.packageName = "com.ss.android.ugc.aweme"
-        arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
-        value.packageName = "com.ss.android.ugc.aweme_oppolianmeng"
-        arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
         }
-      if (dealeridArray.contains("4140")) {
-        value.packageName = "com.xunmeng.pinduoduo"
+
+        val mapData_oppolianmeng= Map("4053" -> "com.ss.android.ugc.aweme",
+          "4143" -> "com.xunmeng.pinduoduo",
+          "4182" -> "me.ele")
+        for (item <- mapData_oppolianmeng) {
+          if (dealeridArray.contains(item._1)) {
+            value.packageName = item._2
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
-        value.packageName = "com.xunmeng.pinduoduo_oppoziyou"
+            value.packageName = item._2 + "_oppolianmeng"
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
           }
-      if (dealeridArray.contains("4143")) {
-        value.packageName = "com.xunmeng.pinduoduo"
-        arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
-        value.packageName = "com.xunmeng.pinduoduo_oppolianmeng"
-        arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
         }
 
         val mapData_oppocn: Map[Int, String] = Map(4060 -> "com.ss.android.ugc.aweme_oppoziyou_notinstall",
@@ -405,7 +390,11 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
           4141 -> "com.xunmeng.pinduoduo_oppoziyou_notinstall",
           4142 -> "com.xunmeng.pinduoduo_oppoziyou_hist_notinstall",
           4144 -> "com.xunmeng.pinduoduo_oppolianmeng_hist1year_notinstall",
-        4145 -> "com.xunmeng.pinduoduo_oppolianmeng_histhalfyear_notinstall")
+          4145 -> "com.xunmeng.pinduoduo_oppolianmeng_histhalfyear_notinstall",
+          4184 -> "me.ele_oppoziyou_hist_notinstall",
+          4185 -> "me.ele_oppoziyou_notinstall",
+          4180 -> "me.ele_oppolianmeng_notinstall",
+          4181 -> "me.ele_oppolianmeng_hist_notinstall")
         for (item <- mapData_oppocn) {
           if (dealeridArray.contains(item._1.toString)) {
             value.packageName = item._2
@@ -432,15 +421,9 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
           3160 -> "com.tencent.news")
         for (item <- mapData) {
           if (dealeridArray.contains(item._1.toString)) {
-          /*
-          arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
-            mapData(num), androidId, time, segment, region)
-          arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
-            mapData(num) + "_oppo", androidId, time, segment, region)
-          */
             value.packageName = item._2
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
-          value.packageName = item._2+"_oppo"
+            value.packageName = item._2 + "_oppo"
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
           }
         }
@@ -449,26 +432,20 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
           2773 -> "com.jingdong.app.mallr_notinstall_oppo")
         for (item <- mapData) {
           if (dealeridArray.contains(item._1.toString)) {
-          /*
-          arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
-            mapData(num), androidId, time, segment, region)
-          */
             value.packageName = item._2
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
           }
         }
-
+      }
     }
     if ("inmobi".equals(exchanges)) {
+      if("android".equalsIgnoreCase(platform)) {
         if (dealeridArray.contains("1594807676568")) {
-        /*
-        arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
-          "com.taobao.taobao_inmobi", androidId, time, segment, region)
-        */
           value.packageName = "com.taobao.taobao_inmobi"
           arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
         }
       }
+    }
     //2020.11.26 增加bes入库需求
     // wiki   https://confluence.mobvista.com/pages/viewpage.action?pageId=47976499
     //2021.04.22 添加adx（exchanges字段）为bes且dealid=100188，则伪包名为com.tencent.news_bes和com.tencent.news
@@ -484,48 +461,50 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
       "100188" -> "com.tencent.news",
       "100310" -> "com.taobao.litetao",
       "100203" -> "com.ss.android.ugc.aweme",
-      "00251" -> "com.xunmeng.pinduoduo")
+      "00251" -> "com.xunmeng.pinduoduo",
+      "100304" -> "me.ele")
 
     val mapData_ios_bes = Map("100197" -> "id387682726",
       "100345" -> "id333206289",
       "100344" -> "id1340376323",
-      "100353" -> "id1044283059")
+      "100353" -> "id1044283059",
+      "100368" -> "id507161324")
 
     if ("bes".equals(exchanges)) {
+      if("android".equalsIgnoreCase(platform)) {
         for (item <- mapData_bes) {
           if (dealeridArray.contains(item._1)) {
-          /*
-          arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
-            mapData_bes(num), androidId, time, segment, region)
-          arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
-            mapData_bes(num) + "_bes", androidId, time, segment, region)
-          */
             value.packageName = item._2
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
-          value.packageName = item._2+"_bes"
+            value.packageName = item._2 + "_bes"
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
           }
         }
+      }
+      if("ios".equalsIgnoreCase(platform)) {
         for (item <- mapData_ios_bes) {
           if (dealeridArray.contains(item._1)) {
             value.packageName = item._2
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
-          value.packageName = item._2+"2021090301"
+            if("100368".equalsIgnoreCase(item._1)){
+              value.packageName = item._2 + "2021091701"
+            }else{
+              value.packageName = item._2 + "2021090301"
+            }
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
           }
         }
       }
+    }
     //2021.04.28  天级别从dsp请求日志（adn_dsp.log_adn_dsp_request_orc_hour）中拉取exchanges='tencent' 并且appid = 'com.tencent.news',入库伪包名为com.tencent.news_fromtencent
     if ("tencent".equals(exchanges)) {
+      if("android".equalsIgnoreCase(platform)) {
         if (packageName.split("#", -1).contains("com.tencent.news")) {
-        /*
-        arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
-          "com.tencent.news_fromtencent", androidId, time, segment, region)
-        */
           value.packageName = "com.tencent.news_fromtencent"
           arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
         }
       }
+    }
     //2021.06.17  天级别从dsp请求日志（adn_dsp.log_adn_dsp_request_orc_hour）中拉取exchanges='kuaishou' 并且appid 符合下列名称的入库
     val mapData_kuaishou = Map("com.smile.gifmaker" -> "com.smile.gifmaker_fromkuaishou",
       "com.kuaishou.nebula" -> "com.kuaishou.nebula_fromkuaishou",
@@ -547,10 +526,12 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
       3996 -> "com.sankuai.meituan",
       3334 -> "com.ss.android.ugc.aweme",
       112799 -> "com.ss.android.ugc.aweme",
-      7156 -> "com.tencent.news")
+      7156 -> "com.tencent.news",
+      1649 -> "me.ele")
     val mapData_ios_iqiyi = Map(
       3334 -> "id1142110895",
-      112799 -> "id1142110895")
+      112799 -> "id1142110895",
+      1649 -> "id507161324")
     if("iqiyi".equals(exchanges)){
       if("android".equalsIgnoreCase(platform)) {
         for (item <- mapData_iqiyi) {
@@ -567,7 +548,11 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
           if (dealeridArray.contains(item._1.toString)) {
             value.packageName = item._2
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
+            if("1649".equalsIgnoreCase(item._1.toString)){
+              value.packageName = item._2 + "2021091702"
+            }else{
               value.packageName = item._2 + "2021090302"
+            }
             arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
           }
         }
@@ -588,6 +573,17 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
       }
     }
 
+    if ("vivo".equals(exchanges)) {
+      if("android".equalsIgnoreCase(platform)) {
+        if (dealeridArray.contains("ba6e4f5323603561086a985b6fc5ffb4")) {
+          value.packageName = "me.ele"
+          arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
+          value.packageName = "me.ele_vivo"
+          arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
+        }
+      }
+    }
+
     //adx=iqiyi 且os=android的imeiMD5和oaidMD5去重设备， 分别和com.taobao.foractivation.227229和com.taobao.foractivation.227229_oaid做差集   2020.12.15
     //2021.04.22下掉,该需求不需要了
     //    if("iqiyi".equals(exchanges)){

src/main/scala/mobvista/dmp/datasource/dsp/DspOrgLogEtlHoursDemo.scala

@@ -141,7 +141,7 @@ class DspOrgLogEtlHoursDemo extends CommonSparkJob with Serializable {
     val exitId = row.getAs[String]("exitid")
     val exchanges = row.getAs[String]("exchanges")
     var dealerid = ""
-    if("mopub".equalsIgnoreCase(exchanges)  || "oppocn".equalsIgnoreCase(exchanges)  || "inmobi".equalsIgnoreCase(exchanges) || "bes".equalsIgnoreCase(exchanges)  || "iqiyi".equalsIgnoreCase(exchanges)  ){
+    if("mopub".equalsIgnoreCase(exchanges)  || "oppocn".equalsIgnoreCase(exchanges)  || "inmobi".equalsIgnoreCase(exchanges) || "bes".equalsIgnoreCase(exchanges)  || "iqiyi".equalsIgnoreCase(exchanges) || "vivo".equalsIgnoreCase(exchanges) ){
       val ext3 = row.getAs[String]("ext3")
       if(StringUtils.isNotBlank(ext3) && ext3.startsWith("{")){
         try{