package mobvista.dmp.datasource.iqiyi; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.google.common.util.concurrent.ListenableFuture; import mobvista.dmp.util.PropertyUtil; import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.*; import ru.yandex.clickhouse.except.ClickHouseUnknownException; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; /** * @package: mobvista.dmp.datasource.iqiyi * @author: wangjf * @date: 2020/4/28 * @time: 4:44 下午 * @email: jinfeng.wang@mobvista.com * @phone: 152-1062-7698 */ public class DMPServer { private static String serverUrl = PropertyUtil.getProperty("config.properties", "iqiyi.dmp.url"); private static String dspId = PropertyUtil.getProperty("config.properties", "iqiyi.dmp.dsp_id"); private static String token = PropertyUtil.getProperty("config.properties", "iqiyi.dmp.token"); private static String[] audienceTargetId = PropertyUtil.getProperty("config.properties", "iqiyi.dmp.audience_target_id").split(","); public static JSONObject request(String deviceId) { JSONObject authJSONObject = new JSONObject(); authJSONObject.put("dsp_id", dspId); authJSONObject.put("token", token); JSONObject checkJSONObject = new JSONObject(); Set<Integer> set = new HashSet<>(); for (String target_id : audienceTargetId) { set.add(Integer.parseInt(target_id)); } checkJSONObject.put("audience_target_id", set); checkJSONObject.put("iqiyi_cookie_id", deviceId); checkJSONObject.put("is_device_id", true); JSONObject requestObject = new JSONObject(); requestObject.put("auth_header", authJSONObject); requestObject.put("check", checkJSONObject); CloseableHttpClient client = HttpClients.createDefault(); RequestConfig requestConfig = RequestConfig.custom() .setConnectTimeout(2000).setConnectionRequestTimeout(2000) .setSocketTimeout(2000).build(); HttpPost post = new HttpPost(serverUrl); JSONObject jsonObject = new JSONObject(); HttpResponse response; try { post.setConfig(requestConfig); post.setEntity(new StringEntity(requestObject.toJSONString())); response = client.execute(post); BufferedReader rd = new BufferedReader( new InputStreamReader(response.getEntity().getContent())); StringBuilder result = new StringBuilder(); String line; while ((line = rd.readLine()) != null) { result.append(line); } jsonObject = JSON.parseObject(result.toString()); } catch (Exception e) { try { post.setConfig(requestConfig); post.setEntity(new StringEntity(requestObject.toJSONString())); response = client.execute(post); BufferedReader rd = new BufferedReader( new InputStreamReader(response.getEntity().getContent())); StringBuilder result = new StringBuilder(); String line; while ((line = rd.readLine()) != null) { result.append(line); } jsonObject = JSON.parseObject(result.toString()); } catch (IOException ex) { // System.out.println("IOException -->> " + ex.getMessage()); jsonObject.put("status", -1); } } finally { post.abort(); } return jsonObject; } public static org.asynchttpclient.ListenableFuture<Response> lahuo(String deviceIds) { final String iqiyiUrl = "http://192.168.0.232:8688/dmp/iqiyi/devicePeopleBatch"; JSONObject requestObject = new JSONObject(); requestObject.put("deviceIds", deviceIds); requestObject.put("encryptType", "1"); requestObject.put("deviceIdType", "1"); AsyncHttpClient asyncHttpClient = new DefaultAsyncHttpClient(new DefaultAsyncHttpClientConfig.Builder() .setConnectTimeout(5000) .setRequestTimeout(5000) .build()); // JSONObject jsonObject = new JSONObject(); /* try { execute = asyncHttpClient.preparePost("").addHeader("Content-Type", "application/json;charset=utf-8").setBody(requestObject.toJSONString()).execute(); Response response = execute.get(5, TimeUnit.SECONDS); if (response.getStatusCode() == HttpConstants.ResponseStatusCodes.OK_200) { jsonObject = JSON.parseObject(response.getResponseBody()); } } catch (Exception e) { // } finally { try { asyncHttpClient.close(); } catch (IOException e) { // } } */ return asyncHttpClient.preparePost(iqiyiUrl) .addHeader("Content-Type", "application/json;charset=utf-8") .setBody(requestObject.toJSONString()).execute(); } public static ResultSet queryIQiYi(Connection connection, String dt, int offSet, int size) throws SQLException, InterruptedException { ResultSet resultSet; String sql = "SELECT device_id, device_type, platform FROM dwh.etl_iqiyi_daily_all WHERE dt = '@dt' LIMIT @offSet, @size"; sql = sql.replace("@dt", dt).replace("@offSet", String.valueOf(offSet)).replace("@size", String.valueOf(size)); try { resultSet = connection.createStatement().executeQuery(sql); return resultSet; } catch (ClickHouseUnknownException e) { Thread.sleep(200); resultSet = connection.createStatement().executeQuery(sql); return resultSet; } } public static void insertIQiYi(Connection connection, List<ListenableFuture<IQiYiClass>> futures) throws SQLException, InterruptedException { PreparedStatement preparedStatement = null; try { preparedStatement = connection.prepareStatement("INSERT INTO dwh.etl_iqiyi_install_daily (dt, device_id, device_type, platform, package_name) VALUES (?, ?, ?, ?, ?)"); for (ListenableFuture<IQiYiClass> future : futures) { try { IQiYiClass iQiYiClass = future.get(); preparedStatement.setString(1, iQiYiClass.getDt()); preparedStatement.setString(2, iQiYiClass.getDeviceId()); preparedStatement.setString(3, iQiYiClass.getDeviceType()); preparedStatement.setString(4, iQiYiClass.getPlatform()); preparedStatement.setString(5, iQiYiClass.getPackageName()); preparedStatement.addBatch(); } catch (ExecutionException e) { e.printStackTrace(); } } preparedStatement.executeBatch(); } catch (ClickHouseUnknownException | NullPointerException e) { Thread.sleep(200); assert preparedStatement != null; preparedStatement.executeBatch(); } finally { if (preparedStatement != null) { preparedStatement.close(); } } } }