1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
package mobvista.dmp.datasource.baichuan
import java.text.SimpleDateFormat
import org.apache.commons.lang3.StringUtils
import org.apache.spark.sql.types.{StringType, StructField, StructType}
/**
* @package: mobvista.dmp.datasource.baichuan
* @author: wangjf
* @date: 2019-08-28
* @time: 11:08
* @email: jinfeng.wang@mobvista.com
* @phone: 152-1062-7698
*/
object Constant {
val baichuan_sql: String =
s"""
|SELECT * FROM dwh.baichuan_install_daily_all WHERE dt = '@date' AND package_name != '0'
""".stripMargin
val baichuan_new_sql: String =
s"""
|SELECT device_id, device_type FROM dwh.baichuan_install_daily_all WHERE dt = '@date' AND package_name = '0'
""".stripMargin
val install_sql: String =
s"""
|SELECT device_id, device_type FROM dwh.dm_install_list_v2 WHERE dt = '@dt' AND business = 'ali'
""".stripMargin
def buildPart(partSize: Int): Seq[String] = {
var list: scala.List[String] = scala.List()
for (i <- 0 until partSize) {
list = list.::(s"rand() % ${partSize} = ${i}")
}
list = list.reverse
list
}
val sdf1 = new SimpleDateFormat("yyyy-MM-dd")
val sdf2 = new SimpleDateFormat("yyyyMMdd")
val filter_sql: String =
s"""
|SELECT dev_id, dev_type, platform, CASE WHEN install IS NOT NULL AND install != '' THEN install ELSE '' END AS install
| FROM dwh.ods_dmp_user_info_daily WHERE dt = '@date' AND UPPER(country) = 'CN'
""".stripMargin
val etl_sql: String =
s"""
|SELECT dev_id, '@osId' app_os, '@appId' app_id FROM ods_user_info_daily WHERE platform = '@appOs' AND dev_type = '@devType' AND hasNo(install,'@package')
""".stripMargin
def hasNo(install: String, pkg: String): Boolean = {
var flag = true
val itr = install.toLowerCase.split(",").iterator
while (itr.hasNext && flag) {
val ir = itr.next
val rs = ir.split("\\|")
if (StringUtils.isNotBlank(ir) && rs(0).equals(pkg)) {
flag = false
}
}
flag
}
val schema = StructType(Array(
StructField("device_id", StringType, true),
StructField("device_type", StringType, true)
))
def checkDevice(deviceId: String): Boolean = {
val imeiAllPtn = """^([0-9a-zA-Z])\1{13,16}"""
!deviceId.matches(imeiAllPtn)
}
val indexColumn: Seq[String] = Seq("device_id", "app_id", "app_os", "tag", "update_date")
val orderColumn: Seq[String] = Seq("device_id", "app_id", "app_os", "update_date")
val baichuanIndexColumn: Seq[String] = Seq("device_id", "device_type", "platform", "package_name")
val baichaunOrderColumn: Seq[String] = Seq("device_id", "device_type", "platform", "package_name")
case class BaiChuan(device_id: String, app_id: Int, app_os: Int) extends java.io.Serializable
}