何冠元 / dw-example · 提交

转到一个项目

作者 [wgf]
提交 75e36e2e3b65e31e2ea84f5828b2ebf97bab26e7 1 个父辈 977099fd

分页API优化

内嵌 并排对比
正在显示 12 个修改的文件 包含 429 行增加 和 306 行删除
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/classes" />
</component>
</project>
\ No newline at end of file
... ...
... ... @@ -11,7 +11,7 @@
提供三种数据请求方案
---
- 全量请求：一次http请求获取所有数据，有数量限制，最大1000条。适合小批量数据同步。
- 全量请求：一次http请求获取所有数据，有数量限制，最大3000条。适合小批量数据同步。
- 分页请求：多次http请求获取所有数据，分页参数`pageNumber`, `pageSize(取值范围[1,3000])`. 适合中批量数据同步。
... ...
package com.aukey.example.conf;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONException;
import com.alibaba.fastjson.JSONReader;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Supplier;
/**
* 描述：流式读取帮助类
* 创建者: wgf
* 创建时间：2020年5月20日 10:03:15
**/
public class StreamReaderHandler {
private static Logger log = LoggerFactory.getLogger(StreamReaderHandler.class);
private StreamReaderHandler() {
}
/**
* @param dwDataApi dw数据请求接口
* @param api 平台申请的api
* @param paramMap 参数
* @param callback 业务回调
* @param batchSize 业务回调数据量大小，参考值 [500,5000]
* @param constructor 实体的构造函数
* @throws Exception
*/
public static <T> void reader(String dwDataApi, String api,
Map<String, Object> paramMap,
Consumer<List<T>> callback,
int batchSize,
Supplier<T> constructor) throws Exception {
dwDataApi = dwDataApi + api;
Response response = null;
InputStream is = null;
Reader reader = null;
JSONReader jsonReader = null;
try {
response = executeHttpPostRequest(dwDataApi, paramMap);
if (response.code() == 200) {
List<T> container = new ArrayList<>();
int total = 0;
// 从响应中获取流
is = response.body().byteStream();
reader = new InputStreamReader(is);
jsonReader = new JSONReader(reader);
// 开始读取
jsonReader.startArray();
while (jsonReader.hasNext()) {
try {
T t = constructor.get();
jsonReader.readObject(t);
container.add(t);
} catch (JSONException ex) {
Object o = jsonReader.readObject();
log.info("json数据转换异常：{}", o.toString());
}
if (container.size() == batchSize) {
callback.accept(container);
total += batchSize;
log.info("{} 流式读取已读条数：{}", dwDataApi, total);
container.clear();
}
}
// 处理最后一批不足 batchSize 的数据
if (container.size() > 0) {
callback.accept(container);
total += container.size();
log.info("{} 流式读取已读条数：{}", dwDataApi, total);
container.clear();
}
//结束读取
jsonReader.endArray();
} else {
log.info("流式读取异常 [ url：{} ] [ code：{} ]", dwDataApi, response.code());
}
} finally {
if (Objects.nonNull(jsonReader)) {
jsonReader.close();
}
if (Objects.nonNull(reader)) {
reader.close();
}
if (Objects.nonNull(is)) {
is.close();
}
if (Objects.nonNull(response)) {
response.close();
}
}
}
protected static Response executeHttpPostRequest(String url, Map<String, Object> paramMap) throws Exception {
OkHttpClient client = new OkHttpClient.Builder()
.connectTimeout(60 * 30 * 2, TimeUnit.SECONDS)
.readTimeout(60 * 1, TimeUnit.SECONDS)
.build();
StringBuilder sb = new StringBuilder(url);
if (paramMap != null && !paramMap.isEmpty()) {
boolean isFirst = true;
for (Map.Entry<String, Object> entry : paramMap.entrySet()) {
if (Objects.isNull(entry.getValue())) {
continue;
}
if (isFirst) {
sb.append("?");
isFirst = false;
} else {
sb.append("&");
}
// 兼容参数特殊符号
String value = String.valueOf(entry.getValue());
value = URLEncoder.encode(value, "UTF-8");
value = value.replaceAll("\\+", "%20");
sb.append(entry.getKey())
.append("=")
.append(value);
}
url = sb.toString();
}
Request request = new Request.Builder().url(url).build();
Response response = client.newCall(request).execute();
return response;
}
}
package com.aukey.example.conf;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.context.request.async.TimeoutCallableProcessingInterceptor;
import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
/**
* @author: wgf
* @create: 2020-05-19 12:56
* @description:
**/
//@Configuration
public class WebMvcConfig implements WebMvcConfigurer {
@Override
public void configureAsyncSupport(final AsyncSupportConfigurer configurer) {
configurer.setDefaultTimeout(30 * 60 * 1000);
configurer.registerCallableInterceptors(timeoutInterceptor());
}
@Bean
public TimeoutCallableProcessingInterceptor timeoutInterceptor() {
return new TimeoutCallableProcessingInterceptor();
}
}
package com.aukey.example.constant;
/**
* @author: wgf
* @create: 2020-05-21 10:36
* @description: 这里面配置的API一定要是授权过的，否则获取不了数据
**/
public interface DwApi {
/**
* 汇率API
*/
String CURRENCY_SET_API = "/base/currency_set";
/**
* FBA-库存快照
*/
String FBA_FULFILLMENT_CURRENT_INVENTORY_VIEW_API = "/stock/fba_fulfillment_current_inventory_view";
}
... ...
package com.aukey.example.job;
import com.aukey.example.util.DwUtil;
import com.aukey.example.util.DwHelperUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
... ... @@ -45,6 +45,6 @@ public class TokenJob {
Map<String, Object> params = new HashMap<>();
params.put("appSecret", this.appSecret);
DwUtil.doGet(dwTokenApi, params);
DwHelperUtil.doGet(dwTokenApi, params);
}
}
... ...
package com.aukey.example.util;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONException;
import com.alibaba.fastjson.JSONReader;
import com.alibaba.fastjson.TypeReference;
import com.aukey.example.vo.DwParamVo;
import com.aukey.example.web.TokenController;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StringUtils;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
/**
* @author: wgf
* @create: 2020-05-13 11:21
* @description:
* @description: dw帮助工具类
*
* 关于 java url 编码问题
* Java官方的URLEncoder.encode 实际上是为了post请求的content-type为x-www-form-urlencoded来设计的
* 在进行特殊参数转义的时候会将空格转为 +
* 但是在 RFC1738、RFC2396协议中规定，GET请求的空格转为为 %20
* 所以在发送GET请求的特殊参数中存在空格必须 先URLEncoder.encode 然后再用 %20替换掉所有+
* 所以在发送GET请求的特殊参数中存在空格必须 先URLEncoder.encode 然后再用 %20 替换掉所有 + 号
**/
public class DwUtil {
private DwUtil() {
public class DwHelperUtil {
private static Logger log = LoggerFactory.getLogger(DwHelperUtil.class);
private DwHelperUtil() {
}
/**
... ... @@ -46,36 +63,7 @@ public class DwUtil {
String result = null;
try {
StringBuilder sb = new StringBuilder(urlStr);
if (params != null && !params.isEmpty()) {
boolean isFirst = true;
for (Map.Entry<String, Object> entry : params.entrySet()) {
if (Objects.isNull(entry.getValue())) {
continue;
}
if (isFirst) {
sb.append("?");
isFirst = false;
} else {
sb.append("&");
}
// 兼容参数特殊符号
String value = String.valueOf(entry.getValue());
value = URLEncoder.encode(value, "UTF-8");
value = value.replaceAll("\\+", "%20");
sb.append(entry.getKey())
.append("=")
.append(value);
}
urlStr = sb.toString();
}
urlStr = jointUrl(urlStr, params);
URL url = new URL(urlStr);
connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
... ... @@ -131,4 +119,216 @@ public class DwUtil {
public static String doGet(String dwDoMain, String api, Map<String, Object> params) {
return doGet(dwDoMain + api, params);
}
/**
* url参数拼接
*
* @param url
* @param params
* @return
*/
public static String jointUrl(String url, Map<String, Object> params) throws UnsupportedEncodingException {
StringBuilder sb = new StringBuilder(url);
if (params != null && !params.isEmpty()) {
boolean isFirst = true;
for (Map.Entry<String, Object> entry : params.entrySet()) {
if (Objects.isNull(entry.getValue())) {
continue;
}
if (isFirst) {
sb.append("?");
isFirst = false;
} else {
sb.append("&");
}
// 兼容参数特殊符号
String value = String.valueOf(entry.getValue());
value = URLEncoder.encode(value, "UTF-8");
value = value.replaceAll("\\+", "%20");
sb.append(entry.getKey())
.append("=")
.append(value);
}
url = sb.toString();
}
return url;
}
/**
* 分页查询
*
* @param callBack 回调方法，需要返回总页数
* @param dwDataApi dw获取数据接口
* @param api 申请的api
* @param paramVo 请求参数
* @param typeReference json字符串转实体引用类型
* @param <T>
*/
public static <T> void pageReader(Function<T, Integer> callBack,
String dwDataApi,
String api,
DwParamVo paramVo,
TypeReference<T> typeReference) {
// 当前页
int currentPageNumber = 0;
// 总页数
int totalPageNum = 1;
// 分页请求
do {
currentPageNumber++;
// 每次请求获取最新token
paramVo.setToken(TokenController.getCurrentToken());
paramVo.setPageNumber(currentPageNumber);
String jsonStr = DwHelperUtil.doGet(dwDataApi, api, paramVo.toMap());
if (StringUtils.isEmpty(jsonStr)) {
throw new RuntimeException(String.format("API %s 调用失败", dwDataApi + api));
}
// json解析为对象
T t = JSON.parseObject(jsonStr, typeReference);
// 回调函数获取总页数
int total = callBack.apply(t);
if (currentPageNumber == 1) {
totalPageNum = calculatePage(total, paramVo.getPageSize());
}
log.info("========== 获取API：{} 第：{}页数据 ==========", dwDataApi + api, currentPageNumber);
} while (currentPageNumber < totalPageNum);
}
/**
* 总页数计算
*
* @param total
* @param pageSize
* @return
*/
private static int calculatePage(int total, int pageSize) {
int result = 1;
if (total % pageSize == 0) {
result = total / pageSize;
} else {
result = total / pageSize + 1;
}
return result;
}
/**
* 流式读取
*
* @param dwDataApi dw数据请求接口
* @param api 平台申请的api
* @param paramMap 参数
* @param callback 业务回调
* @param batchSize 业务回调数据量大小，参考值 [500,5000]
* @param constructor 实体的构造函数
* @throws Exception
*/
public static <T> void streamReader(String dwDataApi, String api,
Map<String, Object> paramMap,
Consumer<List<T>> callback,
int batchSize,
Supplier<T> constructor) throws Exception {
dwDataApi = dwDataApi + api;
Response response = null;
InputStream is = null;
Reader reader = null;
JSONReader jsonReader = null;
try {
response = executeHttpPostRequest(dwDataApi, paramMap);
if (response.code() == 200) {
List<T> container = new ArrayList<>();
int total = 0;
// 从响应中获取流
is = response.body().byteStream();
reader = new InputStreamReader(is);
jsonReader = new JSONReader(reader);
// 开始读取
jsonReader.startArray();
while (jsonReader.hasNext()) {
try {
T t = constructor.get();
jsonReader.readObject(t);
container.add(t);
} catch (JSONException ex) {
Object o = jsonReader.readObject();
log.info("json数据转换异常：{}", o.toString());
}
if (container.size() == batchSize) {
callback.accept(container);
total += batchSize;
log.info("{} 流式读取已读条数：{}", dwDataApi, total);
container.clear();
}
}
// 处理最后一批不足 batchSize 的数据
if (container.size() > 0) {
callback.accept(container);
total += container.size();
log.info("{} 流式读取已读条数：{}", dwDataApi, total);
container.clear();
}
//结束读取
jsonReader.endArray();
} else {
log.info("流式读取异常 [ url：{} ] [ code：{} ]", dwDataApi, response.code());
}
} finally {
if (Objects.nonNull(jsonReader)) {
jsonReader.close();
}
if (Objects.nonNull(reader)) {
reader.close();
}
if (Objects.nonNull(is)) {
is.close();
}
if (Objects.nonNull(response)) {
response.close();
}
}
}
protected static Response executeHttpPostRequest(String url, Map<String, Object> paramMap) throws Exception {
OkHttpClient client = new OkHttpClient.Builder()
.connectTimeout(60 * 30 * 2, TimeUnit.SECONDS)
.readTimeout(60 * 1, TimeUnit.SECONDS)
.build();
url = jointUrl(url, paramMap);
Request request = new Request.Builder().url(url).build();
Response response = client.newCall(request).execute();
return response;
}
}
... ...
... ... @@ -7,9 +7,14 @@ import java.util.Map;
/**
* @author: wgf
* @create: 2020-05-13 12:01
* @description:
* @description: API查询实体
**/
public class DwParamVo {
// TODO 返回数据类型，待扩展
public static final String OBJ = "OBJ";
public static final String SL = "SL";
public DwParamVo() {
}
... ... @@ -39,20 +44,27 @@ public class DwParamVo {
private String multiFields;
/**
* 偏移量（可不传 如果传递offset则必传limit）
* 页码，启始页为1（可不传 如果传递offset则必传limit）
*/
private Integer offset;
private Integer pageNumber;
/**
* 限制条数（可不传 如果传递limit则必传offset）
*/
private Integer limit;
private Integer pageSize;
/**
* 是否流式读写(Y/N)
* 是否流式读写(非流式可不传 Y/N)
*/
private String stream;
/**
* 可不传，默认OBJ
* OBJ: 使用PageVo对象
* SL: 使用
*/
private String dataStructure;
public String getAppId() {
return appId;
}
... ... @@ -85,20 +97,20 @@ public class DwParamVo {
this.multiFields = multiFields;
}
public Integer getOffset() {
return offset;
public Integer getPageNumber() {
return pageNumber;
}
public void setOffset(Integer offset) {
this.offset = offset;
public void setPageNumber(Integer pageNumber) {
this.pageNumber = pageNumber;
}
public Integer getLimit() {
return limit;
public Integer getPageSize() {
return pageSize;
}
public void setLimit(Integer limit) {
this.limit = limit;
public void setPageSize(Integer pageSize) {
this.pageSize = pageSize;
}
public String getStream() {
... ... @@ -112,4 +124,12 @@ public class DwParamVo {
public Map<String, Object> toMap() {
return BeanMap.create(this);
}
public String getDataStructure() {
return dataStructure;
}
public void setDataStructure(String dataStructure) {
this.dataStructure = dataStructure;
}
}
... ...
... ... @@ -27,7 +27,7 @@ public class DwResultVo<T> {
/**
* 返回数据
*/
private List<T> data;
private T data;
public boolean isSuccess() {
return success;
... ... @@ -53,11 +53,11 @@ public class DwResultVo<T> {
this.code = code;
}
public List<T> getData() {
public T getData() {
return data;
}
public void setData(List<T> data) {
public void setData(T data) {
this.data = data;
}
}
... ...
package com.aukey.example.vo;
import io.swagger.annotations.ApiModelProperty;
import java.io.Serializable;
import java.util.List;
/**
* 分页对象
*
* @author 吴耿锋
* @version 2018年5月11日
*/
public class PageVo<T> implements Serializable {
@ApiModelProperty("页数")
private int pageNumber;
@ApiModelProperty("每页条数")
private int pageSize;
@ApiModelProperty("总条数")
private int total;
@ApiModelProperty("每页数据")
private List<T> data;
public int getPageNumber() {
return pageNumber;
}
public void setPageNumber(int pageNumber) {
this.pageNumber = pageNumber;
}
public int getPageSize() {
return pageSize;
}
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
public int getTotal() {
return total;
}
public void setTotal(int total) {
this.total = total;
}
public List<T> getData() {
return data;
}
public void setData(List<T> data) {
this.data = data;
}
}
\ No newline at end of file
... ...
... ... @@ -2,24 +2,25 @@ package com.aukey.example.web;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.TypeReference;
import com.aukey.example.conf.StreamReaderHandler;
import com.aukey.example.constant.DwApi;
import com.aukey.example.entity.CurrencySet;
import com.aukey.example.entity.FbaFulfillmentCurrentInventoryView;
import com.aukey.example.util.DwUtil;
import com.aukey.example.util.DwHelperUtil;
import com.aukey.example.vo.DwParamVo;
import com.aukey.example.vo.DwResultVo;
import com.aukey.example.vo.PageVo;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
/**
... ... @@ -34,7 +35,7 @@ import java.util.function.Supplier;
* ****************************************************************************************
* <p>
* Demo 提供三种数据拉取方式，
* 第一种：直接调用API适合[1, 100000]数据读取
* 第一种：直接调用API适合[1, 3000]数据读取
* 第二种：分页适合[1, 1000000]数据读取，数据基数过大深层分页会影响查询性能，并且客户端需要不断发起请求
* 第三种：流式读取适合[1, 10000000]数据读取，不存在深层分页性能影响，并且客户端只需要发起一次请求
**/
... ... @@ -43,17 +44,14 @@ import java.util.function.Supplier;
public class TestApiController {
private Logger log = LoggerFactory.getLogger(TestApiController.class);
// API需要在傲基数仓申请授权
public static final String CURRENCY_SET_API = "/base/currency_set";
public static final String FBA_FULFILLMENT_CURRENT_INVENTORY_VIEW_API = "/stock/fba_fulfillment_current_inventory_view";
@Value("${dwDataApi:null}")
private String dwDataApi;
@Value("${dwAppId:null}")
private String appId;
@ApiOperation(value = "获取前100条汇率 DEMO")
@ApiOperation(value = "小批量读取 DEMO")
@GetMapping("/query_all")
public void queryAll() {
... ... @@ -62,70 +60,68 @@ public class TestApiController {
// 添加查询条件
paramVo.setQueryCondition("WHERE currency_code = 'USD'");
// 指定获取条数
paramVo.setOffset(0);
paramVo.setLimit(100);
// 每页数据条数 dw服务端有限制 取值[1, 3000]
paramVo.setPageSize(3000);
// 调用API
String result = DwUtil.doGet(dwDataApi, CURRENCY_SET_API, paramVo.toMap());
String result = DwHelperUtil.doGet(dwDataApi, DwApi.CURRENCY_SET_API, paramVo.toMap());
if (StringUtils.isEmpty(result)) {
throw new RuntimeException(String.format("API %s 调用失败", CURRENCY_SET_API));
throw new RuntimeException(String.format("API %s 调用失败", DwApi.CURRENCY_SET_API));
}
// json解析为对象
DwResultVo<CurrencySet> resultVo = JSON.parseObject(result, new TypeReference<DwResultVo<CurrencySet>>() {
DwResultVo<PageVo<CurrencySet>> resultVo = JSON.parseObject(result, new TypeReference<DwResultVo<PageVo<CurrencySet>>>() {
});
log.info("");
log.info("↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓");
log.info("API请求状态：{}", resultVo.getMessage());
log.info("API TOP100 数据：{}", JSON.toJSONString(resultVo.getData()));
log.info("↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑");
log.info("API返回数据：{}", JSON.toJSONString(resultVo.getData().getData()));
log.info("API返回数据两：{}", resultVo.getData().getData().size());
log.info("");
}
@ApiOperation(value = "分页查询 DEMO")
/**
* 适合获取 百万级别的表，大表用分页读取会非常慢
* 大表建议使用流式读取
*/
@ApiOperation(value = "分页读取 DEMO")
@GetMapping("/query_page")
public void queryPage() {
// 构造请求参数
DwParamVo paramVo = new DwParamVo(this.appId, TokenController.getCurrentToken());
// 页数
int pageNum = 0;
// 每页数据大小
final int pageSize = 1000;
// 当前页数据
int currentPageSize;
// 分页请求
do {
paramVo.setToken(TokenController.getCurrentToken());
paramVo.setOffset(pageNum * pageSize);
paramVo.setLimit(pageSize);
// TODO 自定义查询条件
String result = DwUtil.doGet(dwDataApi, CURRENCY_SET_API, paramVo.toMap());
if (StringUtils.isEmpty(result)) {
throw new RuntimeException(String.format("API %s 调用失败", CURRENCY_SET_API));
}
// json解析为对象
DwResultVo<CurrencySet> resultVo = JSON.parseObject(result, new TypeReference<DwResultVo<CurrencySet>>() {
});
currentPageSize = CollectionUtils.isEmpty(resultVo.getData()) ? 0 : resultVo.getData().size();
pageNum++;
log.info("========== 获取API：{} 第：{}页数据 数据条数：{} ==========", CURRENCY_SET_API, pageNum + 1, currentPageSize);
// TODO 自定义实现持久化逻辑
} while (currentPageSize == pageSize);
// 每页数据条数 dw服务端有限制 取值[1, 3000]
paramVo.setPageSize(1000);
// TODO 自定义查询条件
// json字符串转实体引用类型
TypeReference typeReference = new TypeReference<DwResultVo<PageVo<CurrencySet>>>() {
};
// 回调函数
Function<DwResultVo<PageVo<CurrencySet>>, Integer> callback = (DwResultVo<PageVo<CurrencySet>> resultVo) -> {
List<CurrencySet> dataList = resultVo.getData().getData();
System.out.println(dataList.size());
// TODO 业务逻辑在这里实现
// ... more
// mapper.insertList(dataList)
// 需要返回总页数
return resultVo.getData().getTotal();
};
// 使用分页API
DwHelperUtil.pageReader(callback,
dwDataApi,
DwApi.CURRENCY_SET_API,
paramVo,
typeReference);
}
/**
* 流式读取适合百/千万级别的内网数据同步，本机测试
* 流式读取适合百/千万级别的表内网数据同步，本机测试
* <p>
* 网络环境：局域网
* 数据源 ：华为dws
... ... @@ -138,10 +134,10 @@ public class TestApiController {
* 测试报表
* **************************************
* *
* * QPS ：1800
* * 耗时 ：100W / 5分钟
* * CPU毛刺 ：1% - 5% 波动
* * 内存毛刺：50Mb - 150Mb 波动
* * 每秒获取条数 ：1700
* * 耗时 ：100W / 5分钟
* * CPU毛刺 ：1% - 5% 波动
* * 内存毛刺 ：50Mb - 150Mb 波动
* *
* **************************************
*/
... ... @@ -156,14 +152,14 @@ public class TestApiController {
// 设置为流式读取
paramVo.setStream("Y");
// TODO 自定义查询条件
paramVo.setOffset(3000000);
// 定义实体构造函数
Supplier<FbaFulfillmentCurrentInventoryView> constructor = FbaFulfillmentCurrentInventoryView::new;
// 流式读取回调函数
Consumer<List<FbaFulfillmentCurrentInventoryView>> callBack = (List<FbaFulfillmentCurrentInventoryView> list) -> {
// TODO 自定义实现持久化逻辑
System.out.println(list.size());
// TODO 业务逻辑在这里实现
// ... more
// mapper.insertList(list)
};
... ... @@ -173,9 +169,9 @@ public class TestApiController {
int batchSize = 5000;
// 使用 StreamAPI
StreamReaderHandler.reader(
DwHelperUtil.streamReader(
dwDataApi,
FBA_FULFILLMENT_CURRENT_INVENTORY_VIEW_API,
DwApi.FBA_FULFILLMENT_CURRENT_INVENTORY_VIEW_API,
paramVo.toMap(),
callBack,
batchSize,
... ...
... ... @@ -28,7 +28,7 @@ public class TokenController {
/**
* 当前token
*/
private static String currentToken = "";
private static String currentToken = null;
/*********************************************************************
... ... @@ -36,7 +36,7 @@ public class TokenController {
* 例如APP里回调地址为 http://localhost:8099/token/receive *
* TokenJob定时任务定时请求数据仓库获取授权API后 *
* 数据仓库回调 http://localhost:8099/token/receive，并且传递token参数 *
* 获取token后，可以放在 mysql,redis等。这里DEMO取巧放在内存中。 *
* 获取token后，可以放在 mysql,redis等。DEMO就直接放在内存中。 *
********************************************************************/
@PostMapping("/receive")
public void receive(String token) {
... ... @@ -45,10 +45,10 @@ public class TokenController {
}
public static String getCurrentToken() {
for (int i = 0; i < 5; i++) {
for (int i = 0; i < 3; i++) {
if (StringUtils.isEmpty(currentToken)) {
try {
TimeUnit.SECONDS.sleep(2);
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
}
... ...