fix

azkaban/mysql2hive.sh

@@ -15,7 +15,8 @@ sqoop import "-Dorg.apache.sqoop.splitter.allow_text_splitter=true" --connect jd
 base_dir="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/dim_package_tags"
 
 sql="INSERT OVERWRITE DIRECTORY '${base_dir}/${date_path}'
-        SELECT a.package_name,a.platform,b.tag_type,b.first_tag,b.second_tag,b.comment_cn
+        SELECT DISTINCT * FROM
+            (SELECT a.package_name,a.platform,b.tag_type,b.first_tag,b.second_tag,b.comment_cn
                 FROM (SELECT package_name,lower(platform) platform,feat_id
                     FROM dev.app_tag
                     LATERAL VIEW EXPLODE(SPLIT(REGEXP_REPLACE(REGEXP_REPLACE(SUBSTRING(category,2,LENGTH(category) - 2),'\"',''),' ',''),',')) featTable AS feat_id) a
@@ -23,6 +24,7 @@ sql="INSERT OVERWRITE DIRECTORY '${base_dir}/${date_path}'
             UNION ALL
             SELECT package_name,platform,tag_type,first_tag,second_tag,comment
             FROM dwh.dim_package_tags
+        ) package_tags
     "
 
 hive -e "${sql}"