package com.reyun.service.impl; import com.google.common.base.Splitter; import com.google.common.collect.Lists; import com.reyun.service.IOpenApiService; import com.reyun.util.AwsS3Util; import org.anarres.lzo.LzoAlgorithm; import org.anarres.lzo.LzoDecompressor; import org.anarres.lzo.LzoInputStream; import org.anarres.lzo.LzoLibrary; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.Collections; import java.util.Comparator; import java.util.List; /** * description: * * @author nolan * @date 13/09/2017 */ @Service public class OpenApiServiceImpl implements IOpenApiService { private static Logger logger = LoggerFactory.getLogger(OpenApiServiceImpl.class); @Override public List<Object[]> listInfo4Toutiao(String ds) { String skey = "warehouse/tkio/tkdm.db/tkdm.today_tt_day/ds=" + ds + "/"; final AwsS3Util s3Util = AwsS3Util.getInstance(); List itemList = AwsS3Util.getInstance().getS3Keys("reyuntkio", skey); if (itemList == null || itemList.size() == 0) { return Lists.newArrayList(); } List<Object[]> rtnList = Lists.newArrayList(); for (int i = 0; i < itemList.size(); i++) { String s3key = String.valueOf(itemList.get(i)); rtnList.addAll(readS3File(s3Util, s3key)); } Collections.sort(rtnList, new Comparator<Object[]>() { @Override public int compare(Object[] o1, Object[] o2) { int cp = 0; cp = Long.valueOf(String.valueOf(o2[1])).compareTo(new Long(String.valueOf(o1[1]))); if (cp == 0) { cp = Long.valueOf(String.valueOf(o2[2])).compareTo(new Long(String.valueOf(o1[2]))); } if (cp == 0) { cp = Long.valueOf(String.valueOf(o2[3])).compareTo(new Long(String.valueOf(o1[3]))); } return cp; } }); return rtnList; } private List<Object[]> readS3File(AwsS3Util s3Util, String s3key) { List<Object[]> rtnList = Lists.newArrayList(); InputStream inputStream = null; BufferedReader br = null; try { final byte[] fileBytes = s3Util.downloadBytesFromS3("reyuntkio", s3key); inputStream = new ByteArrayInputStream(fileBytes); if (s3key.endsWith(".lzo_deflate")) { LzoAlgorithm algorithm = LzoAlgorithm.LZO1X; LzoDecompressor deCompressor = LzoLibrary.getInstance().newDecompressor(algorithm, null); LzoInputStream stream = new LzoInputStream(inputStream, deCompressor); br = new BufferedReader(new InputStreamReader(stream)); } else { br = new BufferedReader(new InputStreamReader(inputStream)); } String line = null; while ((line = br.readLine()) != null) { rtnList.add(Splitter.on("\t").trimResults().splitToList(line).toArray()); } } catch (Exception e) { logger.error("读取s3文件错误", e); } finally { try { if (br != null) { br.close(); } if (inputStream != null) { inputStream.close(); } } catch (IOException e) { e.printStackTrace(); } } return rtnList; } }