Commit 408f3263 by WangJinfeng

init id_mapping

parent dfe66020
...@@ -14,7 +14,9 @@ check_await "${ADN_REQUEST_INPUT_PATH}/_SUCCESS" ...@@ -14,7 +14,9 @@ check_await "${ADN_REQUEST_INPUT_PATH}/_SUCCESS"
check_await "${DSP_INPUT_PATH}/_SUCCESS" check_await "${DSP_INPUT_PATH}/_SUCCESS"
OUTPUT_PATH=${ADS_DEVICE_ID_MAPPING}/${date_path} OUTPUT_PATH=${ADS_DEVICE_MID_ID_MAPPING}/${date_path}
RESULT_OUTPUT_PATH=${ADS_DEVICE_ID_MAPPING}/${date_path}
country="US" country="US"
...@@ -28,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.id_mapping.IDMappingGraphx \ ...@@ -28,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.id_mapping.IDMappingGraphx \
--conf spark.default.parallelism=10000 \ --conf spark.default.parallelism=10000 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \ --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 12g --driver-memory 8g --executor-cores 5 --num-executors 100 \ --master yarn --deploy-mode cluster --executor-memory 12g --driver-memory 8g --executor-cores 5 --num-executors 100 \
../${JAR} -date ${LOG_TIME} -country ${country} -platform ${platform} -output ${OUTPUT_PATH} -coalesce 500 ../${JAR} -date ${LOG_TIME} -country ${country} -platform ${platform} -output ${OUTPUT_PATH} -result_output ${RESULT_OUTPUT_PATH} -coalesce 500
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
exit 255 exit 255
......
...@@ -31,6 +31,7 @@ class IDMappingGraphx extends CommonSparkJob with Serializable { ...@@ -31,6 +31,7 @@ class IDMappingGraphx extends CommonSparkJob with Serializable {
options.addOption("date", true, "date") options.addOption("date", true, "date")
options.addOption("output", true, "output") options.addOption("output", true, "output")
options.addOption("coalesce", true, "coalesce") options.addOption("coalesce", true, "coalesce")
options.addOption("result_output", true, "result_output")
options options
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment