1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
package mobvista.dmp.util;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.transfer.MultipleFileUpload;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
/**
* @package: mobvista.dmp.util
* @author: wangjf
* @date: 2021/8/3
* @time: 11:22 上午
* @email: jinfeng.wang@mobvista.com
*/
public class AwsUploadFileToS3 {
private static Regions clientRegion = Regions.US_EAST_1;
private static AWSCredentials credentials = new BasicAWSCredentials(
PropertyUtil.getProperty("config.properties", "aws.accessKey"),
PropertyUtil.getProperty("config.properties", "aws.secretKey")
);
private static AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
.withRegion(clientRegion)
.withCredentials(new AWSStaticCredentialsProvider(credentials))
.build();
public void uploadFileList(List<String> file_paths, Integer jobId, String deviceType, String part) {
String bucketName = PropertyUtil.getProperty("config.properties", "aws.s3.bucket");
String keyName = PropertyUtil.getProperty("config.properties", "aws.s3.key") + "/" + jobId + "/" + part + "/" + deviceType;
ArrayList<File> files = new ArrayList<File>();
for (String path : file_paths) {
files.add(new File(path));
}
TransferManager xfer_mgr = TransferManagerBuilder.standard()
.withS3Client(s3Client)
.build();
try {
MultipleFileUpload xfer = xfer_mgr.uploadFileList(bucketName,
keyName, new File(PropertyUtil.getProperty("config.properties", "rtdmp.output.dir")), files);
XferMgrProgress.showTransferProgress(xfer);
XferMgrProgress.waitForCompletion(xfer);
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
}
xfer_mgr.shutdownNow(false);
for (String path : file_paths) {
File deleteFile = new File(path);
// 上传完完后即删除原文件
if (deleteFile.exists()) {
deleteFile.delete();
}
}
}
}