Skip to content

L
lighthouse
Commit 176a69b9 by lijiayu9745
Options

Initial commit

parents
Showing with 1322 additions and 0 deletions
.idea/compiler.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="lighthouse" />
<module name="openapi_v1" />
</profile>
</annotationProcessing>
</component>
</project>
\ No newline at end of file
.idea/encodings.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$" charset="UTF-8" />
</component>
</project>
\ No newline at end of file
.idea/inspectionProfiles/Project_Default.xml 0 → 100644
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="Duplicates" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
</profile>
</component>
\ No newline at end of file
.idea/misc.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
\ No newline at end of file
.idea/uiDesigner.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Palette2">
<group name="Swing">
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
</item>
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
</item>
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
</item>
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
<initial-values>
<property name="text" value="Button" />
</initial-values>
</item>
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="RadioButton" />
</initial-values>
</item>
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="CheckBox" />
</initial-values>
</item>
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
<initial-values>
<property name="text" value="Label" />
</initial-values>
</item>
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
</item>
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
</item>
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
<preferred-size width="-1" height="20" />
</default-constraints>
</item>
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
</item>
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
</item>
</group>
</component>
</project>
\ No newline at end of file
.idea/vcs.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
HELP.md 0 → 100644
# Getting Started
### Reference Documentation
For further reference, please consider the following sections:
* [Official Apache Maven documentation](https://maven.apache.org/guides/index.html)
### Guides
The following guides illustrate how to use some features concretely:
* [Building a RESTful Web Service](https://spring.io/guides/gs/rest-service/)
* [Serving Web Content with Spring MVC](https://spring.io/guides/gs/serving-web-content/)
* [Building REST services with Spring](https://spring.io/guides/tutorials/bookmarks/)
lighthouse.iml 0 → 100644
This diff is collapsed. Click to expand it.
pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.6.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.reyun.bpu.dmp</groupId>
<artifactId>openapi_v1</artifactId>
<version>lighthouse-1.0.0</version>
<name>openapi_v1</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!--<dependency>-->
<!--<groupId>org.springframework.boot</groupId>-->
<!--<artifactId>spring-boot-starter-test</artifactId>-->
<!--<scope>test</scope>-->
<!--</dependency>-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-log4j</artifactId>
<version>1.3.8.RELEASE</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.31</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.7.1</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId>
<version>1.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
<version>1.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>1.2.1</version>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
<version>1.9.3</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.6</version>
</dependency>
<!-- 连接s3 -->
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
<version>2.5.51</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
<!--maven打jar包（无论是scala还是java）时候指定主类的插件-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<mainClass>com.reyun.bpu.dmp.DmpBackgroundApplication</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
</plugins>
<finalName>lighthouse-1.0.0</finalName>
</build>
</project>
src/.DS_Store 0 → 100644
File added
src/main/java/com/reyun/bpu/dmp/DmpBackgroundApplication.java 0 → 100644
package com.reyun.bpu.dmp;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
@SpringBootApplication
@ComponentScan(basePackages = {"com.reyun.bpu.dmp.controller"})
public class DmpBackgroundApplication {
public static void main(String[] args) {
SpringApplication.run(DmpBackgroundApplication.class, args);
}
}
src/main/java/com/reyun/bpu/dmp/config/CorsConfig.java 0 → 100644
package com.reyun.bpu.dmp.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
import org.springframework.web.filter.CorsFilter;
/**
* 允许跨域请求
*/
@Configuration
public class CorsConfig {
private CorsConfiguration buildConfig() {
CorsConfiguration corsConfiguration = new CorsConfiguration();
corsConfiguration.addAllowedOrigin("*"); // 1
corsConfiguration.addAllowedHeader("*"); // 2
corsConfiguration.addAllowedMethod("*"); // 3
return corsConfiguration;
}
@Bean
public CorsFilter corsFilter() {
UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
source.registerCorsConfiguration("/**", buildConfig()); // 4
return new CorsFilter(source);
}
}
src/main/java/com/reyun/bpu/dmp/controller/UploadController.java 0 → 100644
package com.reyun.bpu.dmp.controller;
import com.reyun.bpu.dmp.utils.CodeEnum;
import com.reyun.bpu.dmp.utils.FileUtil;
import com.reyun.bpu.dmp.utils.Result;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @program: openapi_v1
*
* @description:
*
* @author: Leo
*
* @create: 2019-06-28 14:28
**/
@RestController
public class UploadController {
private Logger logger = LoggerFactory.getLogger(this.getClass());
@RequestMapping("/helloworld")
public Result<String> helloworld(String username,String password) {
System.out.println("username is:"+username);
System.out.println("username is:"+password);
// return "Hello SpringBoot!!!"; 使用@RestController注解可以直接这样返回
return Result.success();
}
/**
* 标准JSON格式
* @param company 公司名称
* @param multipartFile 文件流
* @return
*/
@RequestMapping(value = "/openapi/supload", method = RequestMethod.POST)
public Result<String> supload(String company,@RequestParam(value = "filePath") MultipartFile multipartFile){
try {
logger.info(company+"====调用自定义接口开始");
new FileUtil().saveFile(multipartFile);
logger.info(company+"====调用自定义接口结束");
return Result.success();
} catch (Exception e) {
logger.error(company+"====调用标准接口失败");
return Result.failed(CodeEnum.FILE_UPLOAD_ERROR);
}
}
/**
* 用户自定义格式
* @param company
* @param multipartFile
* @return
*/
@RequestMapping(value = "/openapi/cupload", method = RequestMethod.POST)
public Result<String> cupload(String company,String fileFormat,@RequestParam(value = "filePath") MultipartFile multipartFile) throws IOException {
try {
logger.info(company+"====调用自定义接口开始");
new FileUtil().saveFile(multipartFile);
logger.info(company+"====调用自定义接口结束");
return Result.success();
} catch (Exception e) {
logger.error(company+"====调用标准接口失败");
return Result.failed(CodeEnum.FILE_UPLOAD_ERROR);
}
}
/**
* 模糊查询
* @param company
* @return
*/
@RequestMapping("/openapi/nameQuery")
public Result<String> nameQuery(String company) {
System.out.println("公司名称：" + company);
List list = new ArrayList<String>();
String rest = "";
list.add("北京阿里巴巴有限公司");
list.add("杭州阿里巴巴有限公司");
list.add("北京热云科技有限公司");
Pattern pattern = Pattern.compile(company);
for (int i = 0; i < list.size(); i++) {
Matcher matcher = pattern.matcher(list.get(i).toString());
if (matcher.find()) {
rest = rest + list.get(i) + ",";
}
}
if (rest.equals("")) {
return Result.failed(CodeEnum.NOT_EXIST_ERROR);
}
else{
String data = rest.substring(0,rest.length()-1);
return Result.success(data);
}
}
}
src/main/java/com/reyun/bpu/dmp/dao/HiveConntion.java 0 → 100644
package com.reyun.bpu.dmp.dao;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
/**
* @program: openapi_v1
*
* @description: hive数据连接
*
* @author: Leo
*
* @create: 2019-07-02 12:11
**/
public class HiveConntion {
private String driverName = "org.apache.hive.jdbc.HiveDriver";
private String url = "jdbc:hive2://10.1.19.74:10000/default";
private String user = "bpu";
private String password = "dfsd";
private Connection conn = null;
private Statement stmt = null;
private ResultSet rs = null;
private String tabName = "";
private String exeSql = "";
public static void main(String[] args) throws Exception {
List<String> list = new ArrayList<String>();
list.add("device_id");
new HiveConntion("test","").createTable();
}
public HiveConntion(String tabName,String exeSql) throws Exception {
try {
this.tabName = tabName;
this.exeSql = exeSql;
Class.forName(driverName);
conn = DriverManager.getConnection(url,user,password);
stmt = conn.createStatement();
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
}
}
public void destory() throws Exception {
if (rs != null) {
rs.close();
}
if (stmt != null) {
stmt.close();
}
if (conn != null) {
conn.close();
}
}
public void showTables() throws Exception {
String sql = "show tables";
rs = stmt.executeQuery(sql);
while (rs.next()) {
System.out.println(rs.getString(1));
}
}
// 加载数据
public void loadData() throws Exception {
String filePath = "";
String sql = "load data local inpath '" + filePath + "' overwrite into table pokes";
stmt.execute(sql);
}
// 创建表
public void createTable() throws Exception {
// String sql = "create table test (tas string) LOCATION 's3://reyunbpu/dmp/datawarehouse/ods/test/'";
stmt.execute(exeSql);
this.destory();
}
}
src/main/java/com/reyun/bpu/dmp/service/FileIoService.java 0 → 100644
package com.reyun.bpu.dmp.service;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
/**
* @program: openapi_v1
*
* @description: s
*
* @author: Leo
*
* @create: 2019-07-02 18:52
**/
public class FileIoService {
public void readFileToS3(String filePath){
try{
RandomAccessFile file = new RandomAccessFile(filePath, "r");
FileChannel fileCh = file.getChannel();
MappedByteBuffer mbbi = fileCh.map(FileChannel.MapMode.READ_ONLY, 0, fileCh.size());
// new S3Util().uploadToS3(filePath,mbbi);
}catch(Exception e){
e.printStackTrace();
}
}
/*
* MappedByteBuffer读写文件
* 测试结果：1.39G读写最短用哪个8.6s
*/
public void MappedByteBufferTest(String company,String filePath) throws IOException {
String targetFile = "/Users/reyun/"+company+".log";
RandomAccessFile rafi = new RandomAccessFile(filePath, "rw");
RandomAccessFile rafo = new RandomAccessFile(targetFile, "rw");
FileChannel fci = rafi.getChannel();
FileChannel fco = rafo.getChannel();
long size = fci.size();
MappedByteBuffer mbbi = fci.map(FileChannel.MapMode.READ_ONLY, 0, size);
MappedByteBuffer mbbo = fco.map(FileChannel.MapMode.READ_WRITE, 0, size);
long start = System.currentTimeMillis();
/**
for (int i = 0; i < size; i++) {
byte b = mbbi.get(i);
mbbo.put(i, b);
}*/
mbbo.put(mbbi);
fci.close();
fco.close();
rafi.close();
rafo.close();
System.out.println("Spend: " + (double) (System.currentTimeMillis() - start) / 1000 + "s");
}
public static void main(String[] args) {
try {
new FileIoService().MappedByteBufferTest("","");
} catch (IOException e) {
e.printStackTrace();
}
}
}
src/main/java/com/reyun/bpu/dmp/service/JsonParse.java 0 → 100644
package com.reyun.bpu.dmp.service;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* @program: openapi_v1
*
* @description: json数据解析
*
* @author: Leo
*
* @create: 2019-07-02 12:00
**/
public class JsonParse {
public static void main(String[] args) throws Exception {
//解析json字符串得到建表字段
// tableInfo1, tableInfo2, tableInfo3分别为建表语句，表字段（真正的解析逻辑都在tableInfo2），表信息如下:
String tableInfo1 = "create external table if not exists " + args[0] + "(";
String tableInfo2 = "";
String tableInfo3 = ") partitioned by (ds string) row format serde 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' \n" +
"with serdeproperties ( \n" +
" 'field.delim'='\\t', \n" +
" 'serialization.format'='\\t') \n" +
"stored as inputformat \n" +
" 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' \n" +
"outputformat\n" +
" 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'\n";
String value = "{\n" +
" \"msgID\":\"1001\",\n" +
" \"gameVersion\": \"2.0\",\n" +
" \"OS\": \"android\",\n" +
" \"accountID\": \"xxx\",\n" +
" \"level\": 15,\n" +
" \"gameServer\": \"金戈铁马\",\n" +
" \"orderID\": \"xxx\",\n" +
" \"iapID\": \"充值包类型\",\n" +
" \"currencyAmount\": 99,\n" +
" \"currencyType\": \"CNY\",\n" +
" \"virtualCurrencyAmount\": 990,\n" +
" \"paymentType\": \"支付宝\",\n" +
" \"status\": \"request\",\n" +
" \"chargeTime\": 1374732001321,\n" +
" \"mission\": \"新手任务1\"\n" +
"}";
LinkedHashMap<String, String> jsonMap =
JSON.parseObject(value, new TypeReference<LinkedHashMap<String, String>>() {});
for (Map.Entry<String, String> entry : jsonMap.entrySet()) {
System.out.println(entry.getKey() + ":" + entry.getValue());
String ss = entry.getKey();
String begaininfo = entry.getValue().substring(0, 1);
System.out.println(begaininfo);
if (begaininfo.equals("[")) {
tableInfo2 = tableInfo2 + ss + " " + "array<string>,";
}
else if (begaininfo.equals("{")) {
tableInfo2 = tableInfo2 + ss + " " + "map<string,string>,";
}
else {
tableInfo2 = tableInfo2 + ss + " " + "string,";
}
}
tableInfo2 = tableInfo2.substring(0,StringUtils.length(tableInfo2) - 1);
System.out.println("");
String driverName = "org.apache.hive.jdbc.HiveDriver";
String sql = tableInfo1 + tableInfo2 + tableInfo3;
System.out.println(sql);
}
}
src/main/java/com/reyun/bpu/dmp/utils/CodeEnum.java 0 → 100644
package com.reyun.bpu.dmp.utils;
public enum CodeEnum {
SUCCESS("200", "SUCCESS"), //
ERROR("500", "系统异常"), //
PARAMS_ERROR("3001", "参数错误!"), //
EXIST_ERROR("3002", "记录已存在!"), //
NOT_EXIST_ERROR("3003", "记录不存在!"), //
VALIDATE_ERROR("3004", "验证错误!"), //
TOKEN_ERROR("3005", "登录凭证错误!"), //
FILE_UPLOAD_ERROR("3006", "文件上传失败!"),
/** 结束code */
CODE_END("9999", "结束CODE，所有枚举加到此代码之前即可!");
private String code;
private String message;
CodeEnum(String code, String message) {
this.code = code;
this.message = message;
}
// getter and setter
public String getCode() {
return code;
}
public String getMessage() {
return message;
}
}
src/main/java/com/reyun/bpu/dmp/utils/FileUtil.java 0 → 100644
package com.reyun.bpu.dmp.utils;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.multipart.MultipartFile;
import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import java.nio.MappedByteBuffer;
import java.nio.file.Path;
import java.nio.file.Paths;
public class FileUtil {
private static Logger log = LoggerFactory.getLogger(FileUtil.class);
@Value("${s3.save.bucket}")
private String bucket;
@Value("${s3.save.path}")
private String uploads3path;
@Value("${local.path}")
private String localpath;
/**
*
* @param file 文件流
* @return
*/
public void uploadToS3(MultipartFile file) {
try {
//上传S3文件
S3Client s3 = S3Client.builder().credentialsProvider(InstanceProfileCredentialsProvider.builder().build()).build();
log.info("获取文件在EMR服务器上的路径");
Path path = Paths.get(localpath+file.getOriginalFilename());
log.info("文件从EMR写入S3开始");
//1、写文件：这种方式需要用上传的文件先写到本地，在从本地上传到S3，因此换一种流的方式直接写到S3
s3.putObject(PutObjectRequest.builder().bucket(bucket).key(uploads3path).build(), path);
log.info("文件从EMR写入S3结束");
//s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(UPLOADPATH).build(), RequestBody.fromByteBuffer(buffer));
} catch (Exception e) {
log.error("文件写S3失败");
e.printStackTrace();
}
}
/**
* 保存文件，直接以multipartFile形式
* 文件先写EMR本地
* @param multipartFile
* @return 返回文件名
* @throws IOException
*/
public String saveFile(MultipartFile multipartFile) {
File file = new File(localpath);
if (!file.exists()) {
file.mkdirs();
}
try {
FileInputStream input = (FileInputStream) multipartFile.getInputStream();
BufferedOutputStream bos = null;
bos = new BufferedOutputStream(new FileOutputStream(localpath + File.separator + multipartFile.getName()));
byte[] bs = new byte[1024];
int len;
log.info("文件上传EMR开始");
while ((len = input.read(bs)) != -1) {
bos.write(bs, 0, len);
}
log.info("文件上传EMR结束");
bos.flush();
bos.close();
uploadToS3(multipartFile);
} catch (FileNotFoundException e) {
log.error("文件没有找到");
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
log.error("文件写入EMR本地失败");
}
return "文件写入EMR成功";
}
/**
* 文件复制
* @param source
* @param dest
* @throws IOException
*/
@SuppressWarnings("resource")
public static void copyFileUsingFileChannels(File source, File dest) throws IOException {
FileChannel inputChannel = null;
FileChannel outputChannel = null;
try {
inputChannel = new FileInputStream(source).getChannel();
outputChannel = new FileOutputStream(dest).getChannel();
outputChannel.transferFrom(inputChannel, 0, inputChannel.size());
} finally {
inputChannel.close();
outputChannel.close();
}
}
public static String readFile(File file, String charset) {
StringBuilder sb = new StringBuilder();
BufferedReader br = null;
InputStreamReader isr = null;
try {
FileInputStream fis = new FileInputStream(file);
isr = new InputStreamReader(fis, charset);
br = new BufferedReader(isr);
while (br.ready()){
sb.append(br.readLine());
sb.append("\n");
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
br.close();
isr.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return sb.toString();
}
/**
* NIO way
* @throws Exception
*/
public static byte[] toByteArray(String filename) throws Exception {
File f = new File(filename);
if (!f.exists()) {
log.error("文件未找到！" + filename);
throw new IOException("文件未找到");
}
FileChannel channel = null;
FileInputStream fs = null;
try {
fs = new FileInputStream(f);
channel = fs.getChannel();
ByteBuffer byteBuffer = ByteBuffer.allocate((int) channel.size());
while ((channel.read(byteBuffer)) > 0) {
// do nothing
// System.out.println("reading");
}
return byteBuffer.array();
} catch (IOException e) {
throw new IOException("FILE_READING_ERROR");
} finally {
try {
channel.close();
} catch (IOException e) {
throw new IOException("FILE_READING_ERROR");
}
try {
fs.close();
} catch (IOException e) {
throw new IOException("FILE_READING_ERROR");
}
}
}
/**
* 删除目录
*
* @author fengshuonan
* @Date 2017/10/30 下午4:15
*/
public static boolean deleteDir(File dir) {
if (dir.isDirectory()) {
String[] children = dir.list();
for (int i = 0; i < children.length; i++) {
boolean success = deleteDir(new File(dir, children[i]));
if (!success) {
return false;
}
}
}
return dir.delete();
}
/**
* 删除单个文件
*
* @param fileName 要删除的文件的文件名
* @return 单个文件删除成功返回true，否则返回false
*/
public static boolean deleteFile(String fileName) {
File file = new File(fileName);
// 如果文件路径所对应的文件存在，并且是一个文件，则直接删除
if (file.exists() && file.isFile()) {
if (file.delete()) {
log.info("删除单个文件" + fileName + "成功！");
return true;
} else {
log.info("删除单个文件" + fileName + "失败！");
return false;
}
} else {
log.error("删除单个文件失败：" + fileName + "不存在！");
return false;
}
}
public static String getFileNameFromPath(String path){
String fileName = path.trim().substring(path.lastIndexOf("/")+1);
return fileName;
}
public static void main(String[] args) {
String filename= "M00/00/00/wKgAmVmY_HWAU0H-AAAtToIKUp8155.doc";
String fileNameFromPath = getFileNameFromPath(filename);
System.out.println(fileNameFromPath);
}
}
\ No newline at end of file
src/main/java/com/reyun/bpu/dmp/utils/Result.java 0 → 100644
package com.reyun.bpu.dmp.utils;
import java.io.Serializable;
public class Result<T> implements Serializable {
private static final long serialVersionUID = 1L;
/**
* 响应代码
*/
private String code;
/**
* 返回说明
*/
private String msg;
/**
* 返回结果
*/
private T data;
public Result() {
}
/**
*
* @param success
* @param code
* @param msg
* @param data
*/
public Result(String code, String msg, T data) {
super();
this.code = code;
this.msg = msg;
this.data = data;
}
/**
*
* @param success
* @param code
* @param msg
*/
public Result(String code, String msg) {
super();
this.code = code;
this.msg = msg;
}
/**
*
* @param codeEnum
*/
public Result(CodeEnum codeEnum) {
super();
this.code = codeEnum.getCode();
this.msg = codeEnum.getMessage();
}
/**
*
* @param codeEnum
* @param data
*/
public Result(CodeEnum codeEnum, T data) {
super();
this.code = codeEnum.getCode();
this.msg = codeEnum.getMessage();
this.data = data;
}
/**
*
* @param codeEnum
* @param msg
* @param data
*/
public Result(CodeEnum codeEnum, String msg, T data) {
super();
this.code = codeEnum.getCode();
this.msg = msg;
this.data = data;
}
public static <T> Result<T> info(String code, String msg, T data) {
return new Result<T>(code, msg, data);
}
@SuppressWarnings({ "rawtypes", "unchecked" })
public static <T> Result<T> success() {
return new Result(CodeEnum.SUCCESS);
}
@SuppressWarnings({ "rawtypes", "unchecked" })
public static <T> Result<T> success(T data) {
return new Result(CodeEnum.SUCCESS, data);
}
@SuppressWarnings({ "rawtypes", "unchecked" })
public static <T> Result<T> success(T data, String msg) {
return new Result(CodeEnum.SUCCESS.getCode(), msg, data);
}
@SuppressWarnings({ "rawtypes", "unchecked" })
public static <T> Result<T> error(String code, String msg) {
return new Result(code, msg);
}
@SuppressWarnings({ "rawtypes", "unchecked" })
public static <T> Result<T> failed(CodeEnum codeEnum) {
return new Result(codeEnum);
}
@SuppressWarnings({ "rawtypes", "unchecked" })
public static <T> Result<T> failed(CodeEnum codeEnum, String msg) {
return new Result(codeEnum.getCode(), msg);
}
@SuppressWarnings({ "rawtypes", "unchecked" })
public static <T> Result<T> failed(CodeEnum codeEnum, String msg, T data) {
return new Result(codeEnum.getCode(), msg, data);
}
public String getCode() {
return code;
}
public void setCode(String code) {
this.code = code;
}
public String getMsg() {
return msg;
}
public void setMsg(String msg) {
this.msg = msg;
}
public T getData() {
return data;
}
public void setData(T data) {
this.data = data;
}
}
src/main/java/com/reyun/bpu/dmp/utils/S3FileUtil.java 0 → 100644
package com.reyun.bpu.dmp.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import java.io.IOException;
import java.nio.MappedByteBuffer;
import java.nio.file.Path;
import java.nio.file.Paths;
/**
* @program: openapi_v1
*
* @description: a
*
* @author: Leo
*
* @create: 2019-07-02 16:48
**/
public class S3FileUtil {
private final Logger logger = LoggerFactory.getLogger(S3FileUtil.class);
private String BUCKET = "test";
private String UPLOADPATH = "s3://reyunbpu/fox/liyanbin_1986.log";
/**
*
* @param buffer 本地文件流
* @param filePath 本地文件路径
* @return
*/
// public void uploadToS3(String filePath, MappedByteBuffer buffer) {
//
// try {
//
// logger.info(String.format("s3 put , bulid S3Client begin ... params:[bucket=%s,fileKey=%s,localFilePath=%s]", BUCKET, UPLOADPATH, filePath));
//
// //上传S3文件
// S3Client s3 = S3Client.builder().credentialsProvider(InstanceProfileCredentialsProvider.builder().build()).build();
//
// logger.info("s3 build S3Clent OK ! get path begin ... ");
//
// // 上传文件
// Path path = Paths.get(filePath);
//
// logger.info("s3 get path OK ! putObject begin ... ");
//
// //1、写文件：这种方式需要用上传的文件先写到本地，在从本地上传到S3，因此换一种流的方式直接写到S3
// /**s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(uploadPath).build(), path);*/
// //2、写流到s3
// s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(UPLOADPATH)
// .build(), RequestBody.fromByteBuffer(buffer));
// logger.info("s3 putObject end ... ");
//
// } catch (Exception e) {
//
// e.printStackTrace();
// }
// }
//
// /**
// *
// * @param filePath 本地文件路径及名称（/file/images/test.jpg）
// * @param downLoadPath S3存放路径及名称（/yubin/image/aa.jpg）
// * @throws IOException
// */
// public void loadFromS3(String filePath, String downLoadPath) throws IOException {
// //下载S3文件
// try {
// S3Client s3 = S3Client.builder()
// .credentialsProvider(InstanceProfileCredentialsProvider.builder().build())
// .build();
// // 下载文件
// Path path = Paths.get(filePath);
// s3.getObject(GetObjectRequest.builder().bucket(BUCKET).key("/"+downLoadPath).build(),
// ResponseTransformer.toFile(path));
// } catch (Exception e) {
// e.printStackTrace();
// }
// }
}
src/main/resources/application.properties 0 → 100644
#关闭修改文件自动重启
spring.devtools.restart.enabled=true
s3.save.bucket=rydmp
s3.save.path=/test/
local.path=/home/ec2-user/dmp/liyanbin/s3upload
src/main/resources/log4j.properties 0 → 100644
#
log4j.rootCategory=INFO, stdout, file
# root日志输出
log4j.appender.file=org.apache.log4j.DailyRollingFileAppender
log4j.appender.file.file=logs/all.log
log4j.appender.file.DatePattern='.'yyyy-MM-dd
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %5p %c{1}:%L - %m%n
\ No newline at end of file
target/classes/application.properties 0 → 100644
#关闭修改文件自动重启
spring.devtools.restart.enabled=true
s3.save.bucket=rydmp
s3.save.path=/test/
local.path=/home/ec2-user/dmp/liyanbin/s3upload
target/classes/log4j.properties 0 → 100644
#
log4j.rootCategory=INFO, stdout, file
# root日志输出
log4j.appender.file=org.apache.log4j.DailyRollingFileAppender
log4j.appender.file.file=logs/all.log
log4j.appender.file.DatePattern='.'yyyy-MM-dd
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %5p %c{1}:%L - %m%n
\ No newline at end of file
target/maven-archiver/pom.properties 0 → 100644
version=lighthouse-1.0.0
groupId=com.reyun.bpu.dmp
artifactId=openapi_v1
target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst 0 → 100644
com\reyun\bpu\dmp\DmpBackgroundApplication.class
com\reyun\bpu\dmp\utils\Result.class
com\reyun\bpu\dmp\dao\HiveConntion.class
com\reyun\bpu\dmp\service\FileIoService.class
com\reyun\bpu\dmp\service\JsonParse.class
com\reyun\bpu\dmp\utils\CodeEnum.class
com\reyun\bpu\dmp\utils\FileUtil.class
com\reyun\bpu\dmp\controller\UploadController.class
com\reyun\bpu\dmp\service\JsonParse$1.class
com\reyun\bpu\dmp\utils\S3FileUtil.class
com\reyun\bpu\dmp\config\CorsConfig.class
target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst 0 → 100644
D:\lighthouse\src\main\java\com\reyun\bpu\dmp\dao\HiveConntion.java
D:\lighthouse\src\main\java\com\reyun\bpu\dmp\utils\FileUtil.java
D:\lighthouse\src\main\java\com\reyun\bpu\dmp\utils\S3FileUtil.java
D:\lighthouse\src\main\java\com\reyun\bpu\dmp\utils\Result.java
D:\lighthouse\src\main\java\com\reyun\bpu\dmp\service\JsonParse.java
D:\lighthouse\src\main\java\com\reyun\bpu\dmp\DmpBackgroundApplication.java
D:\lighthouse\src\main\java\com\reyun\bpu\dmp\config\CorsConfig.java
D:\lighthouse\src\main\java\com\reyun\bpu\dmp\utils\CodeEnum.java
D:\lighthouse\src\main\java\com\reyun\bpu\dmp\controller\UploadController.java
D:\lighthouse\src\main\java\com\reyun\bpu\dmp\service\FileIoService.java
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment