优化流式读取，将普通流修改为压缩流

@@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.zip.GZIPInputStream;
 
 /**
  * @author: wgf
@@ -40,6 +41,28 @@ import java.util.function.Supplier;
 public class DwHelperUtil {
     private static Logger log = LoggerFactory.getLogger(DwHelperUtil.class);
 
+    private static OkHttpClient client;
+
+    /**
+     * http请求默认超时为7200秒
+     * 可根据同步数据大小自行设置超时时间
+     * @return
+     */
+    public static OkHttpClient getClient() {
+        if (client == null) {
+            synchronized (DwHelperUtil.class) {
+                if (client == null) {
+                    client =  new OkHttpClient.Builder()
+                            .callTimeout(60 * 60 * 2, TimeUnit.SECONDS)
+                            .writeTimeout(60 * 2, TimeUnit.SECONDS)
+                            .readTimeout(60 * 2, TimeUnit.SECONDS)
+                            .build();
+                }
+            }
+        }
+        return client;
+    }
+
     private DwHelperUtil() {
     }
 
@@ -247,10 +270,11 @@ public class DwHelperUtil {
                                         Supplier<T> constructor) throws Exception {
 
         dwDataApi = dwDataApi + api;
-        Response    response   = null;
-        InputStream is         = null;
-        Reader      reader     = null;
-        JSONReader  jsonReader = null;
+        Response        response        = null;
+        InputStream     is              = null;
+        GZIPInputStream gzipInputStream = null;
+        Reader          reader          = null;
+        JSONReader      jsonReader      = null;
 
         try {
             response = executeHttpGetRequest(dwDataApi, paramMap);
@@ -258,10 +282,12 @@ public class DwHelperUtil {
             if (response.code() == 200) {
                 List<T> container = new ArrayList<>();
                 int     total     = 0;
+                long    start     = System.currentTimeMillis();
 
                 // 从响应中获取流
                 is = response.body().byteStream();
-                reader = new InputStreamReader(is);
+                gzipInputStream = new GZIPInputStream(is);
+                reader = new InputStreamReader(gzipInputStream);
                 jsonReader = new JSONReader(reader);
                 // 开始读取
                 jsonReader.startArray();
@@ -286,6 +312,9 @@ public class DwHelperUtil {
                     log.info("{} 流式读取已读条数：{}", dwDataApi, total);
                     container.clear();
                 }
+
+                float consuming = (System.currentTimeMillis() - start) / 1000.f;
+                log.info("流式平均每秒读取条数：{}", total / consuming);
             } else {
                 log.info("流式读取异常 [ url：{} ] [ code：{} ]", dwDataApi, response.code());
             }
@@ -300,6 +329,10 @@ public class DwHelperUtil {
                 jsonReader.close();
             }
 
+            if (Objects.nonNull(gzipInputStream)) {
+                gzipInputStream.close();
+            }
+
             if (Objects.nonNull(reader)) {
                 reader.close();
             }
@@ -316,14 +349,8 @@ public class DwHelperUtil {
 
 
     protected static Response executeHttpGetRequest(String url, Map<String, Object> paramMap) throws Exception {
-        OkHttpClient client = new OkHttpClient.Builder()
-                .connectTimeout(60 * 60 * 2, TimeUnit.SECONDS)
-                .writeTimeout(60 * 2, TimeUnit.SECONDS)
-                .readTimeout(60 * 2, TimeUnit.SECONDS)
-                .build();
-
+        OkHttpClient client = getClient();
         url = jointUrl(url, paramMap);
-
         Request  request  = new Request.Builder().url(url).build();
         Response response = client.newCall(request).execute();
         return response;

@@ -197,7 +197,8 @@ public class TestApiController {
 
 
     /**
-     * 流式读取适合百级别的表内网数据同步，每次数据读取建议在[100000, 1000000]，本机测试
+     * 流式读取适合百级别的表内网数据同步，每次数据读取建议在1千万以内，连接2小时会超时。
+     * 几千万级别的建议按条件分批
      * <p>
      * 网络环境：局域网
      * 数据源  ：华为dws
@@ -210,8 +211,8 @@ public class TestApiController {
      * 测试报表
      * **************************************
      * *
-     * *        每秒获取条数     ：1700
-     * *        耗时             ：100W / 5分钟
+     * *        每秒获取条数     ：3100+
+     * *        数据总将         ：1000W+
      * *        CPU毛刺          ：1% - 5% 波动
      * *        内存毛刺         ：50Mb - 150Mb 波动
      * *