Skip to content

M
mobvista-dmp
Commit d9b73faa by WangJinfeng
Options

fix no file

parent 0808694a
Showing with 1 addition and 1 deletion
src/main/scala/mobvista/dmp/datasource/rtdmp/RTDmpMainPre.scala
...@@ -156,7 +156,7 @@ class RTDmpMainPre extends CommonSparkJob with Serializable { ...@@ -156,7 +156,7 @@ class RTDmpMainPre extends CommonSparkJob with Serializable {
val pathUri = new URI(audData.get(0)._1) val pathUri = new URI(audData.get(0)._1)
if (audData.nonEmpty && FileSystem.get(new URI(s"${pathUri.getScheme}://${pathUri.getHost}"), sc.hadoopConfiguration) if (audData.nonEmpty && FileSystem.get(new URI(s"${pathUri.getScheme}://${pathUri.getHost}"), sc.hadoopConfiguration)
.exists(new Path(audData.get(0)._1.replace("*", "")))) { .exists(new Path(audData.get(0)._1.replace("*", "")))) {
val rdd = sc.newAPIHadoopFile(audData.get(0)._1, fc, kc, vc, sc.hadoopConfiguration) val rdd = sc.newAPIHadoopFile(audData.get(0)._1.replace("*",""), fc, kc, vc, sc.hadoopConfiguration)
val linesWithFileNames = rdd.asInstanceOf[NewHadoopRDD[LongWritable, Text]] val linesWithFileNames = rdd.asInstanceOf[NewHadoopRDD[LongWritable, Text]]
.mapPartitionsWithInputSplit((inputSplit, iterator) => { .mapPartitionsWithInputSplit((inputSplit, iterator) => {
val file = inputSplit.asInstanceOf[FileSplit] val file = inputSplit.asInstanceOf[FileSplit]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment