package mobvista.dmp.clickhouse.realtime import org.apache.commons.lang3.StringUtils /** * @package: mobvista.dmp.clickhouse.realtime * @author: wangjf * @date: 2019-10-23 * @time: 11:34 * @email: jinfeng.wang@mobvista.com * @phone: 152-1062-7698 */ object Constant { val indexColumn: Seq[String] = Seq("device_id", "platform", "country", "install_apps") val orderColumn: Seq[String] = Seq("device_id", "platform") val old2new_sql: String = """ |SELECT UPPER(CONCAT(tag_type, '-', first_tag, '-', second_tag)) tag_code, new_second_id FROM | dwh.dm_old2new_tag """.stripMargin val id_old2new_sql: String = """ |SELECT new_second_id tag_code, tag_id FROM | dwh.dm_old2new_tag WHERE tag_id != '' AND tag_id IS NOT NULL """.stripMargin val statistics_sql: String = """ |SELECT LOWER(device_id) device_id,COLLECT_SET((tag,cnt)) frequency | FROM( | SELECT device_id, getId(CONCAT(tag_type,'-',first_tag,'-',second_tag)) tag, CAST(cnt AS INT) cnt | FROM dwh.dm_device_tag_statistics | WHERE dt = '@date' AND checkDevice(device_id) | ) tmp WHERE tag != '' | GROUP BY LOWER(device_id) """.stripMargin val ods_dmp_user_info_all_sql_distinct: String = """ |SELECT dm_device.device_id, dm_device.platform, dm_device.model, dm_device.osversion os_version, dm_device.country, CAST(COALESCE(dm_device.age,10) AS INT) age, CAST(COALESCE(dm_device.gender,10) AS INT) gender, dm_device.install, dm_device.interest, | dm_device.behavior, dm_active.frequency, active_week.tags tag_week, active_month.tags tag_month, dm_device.update_date | FROM | (SELECT LOWER(dev_id) device_id, platform, model, osversion, CASE WHEN country = 'GB' THEN 'UK' ELSE country END AS country, age, gender, getInstallList(install) install, getInterestList(interest) interest, behavior, update_date | FROM dwh.ods_dmp_user_info_all WHERE dt = '@date' AND update_date >= '@update_date' AND checkDevice(dev_id) | ) dm_device | LEFT JOIN dm_active ON dm_device.device_id = dm_active.device_id | LEFT JOIN (SELECT LOWER(device_id) device_id, MAX(tags) tags FROM dwh.dm_active_tag WHERE dt = '@activeDate' AND part = 'week' GROUP BY lower(device_id)) active_week ON dm_device.device_id = active_week.device_id | LEFT JOIN (SELECT LOWER(device_id) device_id, MAX(tags) tags FROM dwh.dm_active_tag WHERE dt = '@activeDate' AND part = 'month' GROUP BY lower(device_id)) active_month ON dm_device.device_id = active_month.device_id """.stripMargin val readFromCKToS3_sql: String = """ |SELECT UPPER(device_id) device_id FROM @database.@table WHERE dt = '@dt' AND hour = '@hour' AND region = '@region' AND flag > 0 """.stripMargin val readFromCKToCa_sql: String = """ |SELECT LOWER(device_id) device_id, age, gender, country, toString(interest) interest, toString(install_apps) install_apps, frequency FROM @database.@table WHERE dt = '@dt' AND hour = '@hour' AND region = '@region' AND flag > 0 """.stripMargin def getColumns(table: String): String = { table match { case "a" => "device_id, platform, age, country, toString(interest) interest, toString(install) install" case "b" => "" } } def checkDevice(device_id: String): Boolean = { StringUtils.isNotBlank(device_id) } def buildPart(partSize: Int): Seq[String] = { var list: scala.List[String] = scala.List() for (i <- 0 until partSize) { list = list.::(s"CRC32(device_id) % ${partSize} = ${i}") } list = list.reverse list } }