Merge branch 'master' into eureka

eureka
tanghc 4 years ago
commit 31b67516ee
  1. 1
      changelog.md
  2. 25
      sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/GatewayModifyResponseGatewayFilter.java
  3. 21
      sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/ApiArgumentResolver.java
  4. 53
      sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/ByteArrayStreamWrapper.java
  5. 106
      sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/MyServletRequestWrapper.java
  6. 18
      sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/util/OpenUtil.java
  7. 5
      sop-example/sop-story/pom.xml
  8. 27
      sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java
  9. 4
      sop-gateway/src/main/resources/application-dev.properties
  10. 19
      sop-test/src/test/java/com/gitee/sop/test/AllInOneTest.java

@ -4,6 +4,7 @@
- 可定义业务错误码(见`@Open`注解中的`bizCode`属性) - 可定义业务错误码(见`@Open`注解中的`bizCode`属性)
- 文档参数可指定最大长度(使用`@Length(max = xx)`) - 文档参数可指定最大长度(使用`@Length(max = xx)`)
- 修复返回大文本导致的错误(Exceeded limit on max bytes to buffer : 262144)
## 4.0.2 ## 4.0.2

@ -5,6 +5,7 @@ import com.gitee.sop.gatewaycommon.bean.SopConstants;
import com.gitee.sop.gatewaycommon.result.ResultExecutor; import com.gitee.sop.gatewaycommon.result.ResultExecutor;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.reactivestreams.Publisher; import org.reactivestreams.Publisher;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.gateway.filter.GatewayFilterChain; import org.springframework.cloud.gateway.filter.GatewayFilterChain;
import org.springframework.cloud.gateway.filter.GlobalFilter; import org.springframework.cloud.gateway.filter.GlobalFilter;
import org.springframework.cloud.gateway.filter.NettyWriteResponseFilter; import org.springframework.cloud.gateway.filter.NettyWriteResponseFilter;
@ -12,12 +13,16 @@ import org.springframework.cloud.gateway.filter.factory.rewrite.CachedBodyOutput
import org.springframework.cloud.gateway.support.BodyInserterContext; import org.springframework.cloud.gateway.support.BodyInserterContext;
import org.springframework.cloud.gateway.support.DefaultClientResponse; import org.springframework.cloud.gateway.support.DefaultClientResponse;
import org.springframework.core.Ordered; import org.springframework.core.Ordered;
import org.springframework.core.codec.AbstractDataBufferDecoder;
import org.springframework.core.codec.Decoder;
import org.springframework.core.io.buffer.DataBuffer; import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import org.springframework.http.ResponseCookie; import org.springframework.http.ResponseCookie;
import org.springframework.http.client.reactive.ClientHttpResponse; import org.springframework.http.client.reactive.ClientHttpResponse;
import org.springframework.http.codec.DecoderHttpMessageReader;
import org.springframework.http.codec.HttpMessageReader;
import org.springframework.http.server.reactive.ServerHttpResponseDecorator; import org.springframework.http.server.reactive.ServerHttpResponseDecorator;
import org.springframework.util.MultiValueMap; import org.springframework.util.MultiValueMap;
import org.springframework.web.reactive.function.BodyInserter; import org.springframework.web.reactive.function.BodyInserter;
@ -35,6 +40,8 @@ import static org.springframework.cloud.gateway.support.ServerWebExchangeUtils.O
*/ */
public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered { public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered {
@Value("${spring.codec.max-in-memory-size:262144}")
private int maxInMemorySize;
@Override @Override
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@ -60,7 +67,7 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered
//this will prevent exception in case of using non-standard media types like "Content-Type: image" //this will prevent exception in case of using non-standard media types like "Content-Type: image"
httpHeaders.add(HttpHeaders.CONTENT_TYPE, originalResponseContentType); httpHeaders.add(HttpHeaders.CONTENT_TYPE, originalResponseContentType);
ResponseAdapter responseAdapter = new ResponseAdapter(body, httpHeaders); ResponseAdapter responseAdapter = new ResponseAdapter(body, httpHeaders);
DefaultClientResponse clientResponse = new DefaultClientResponse(responseAdapter, ExchangeStrategies.withDefaults()); DefaultClientResponse clientResponse = new DefaultClientResponse(responseAdapter, getExchangeStrategies());
//TODO: flux or mono //TODO: flux or mono
Mono modifiedBody = clientResponse.bodyToMono(inClass) Mono modifiedBody = clientResponse.bodyToMono(inClass)
@ -97,6 +104,22 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered
return chain.filter(exchange.mutate().response(responseDecorator).build()); return chain.filter(exchange.mutate().response(responseDecorator).build());
} }
private ExchangeStrategies getExchangeStrategies() {
ExchangeStrategies exchangeStrategies = ExchangeStrategies.withDefaults();
// 修复返回大文本数据报org.springframework.core.io.buffer.DataBufferLimitException: Exceeded limit on max bytes to buffer : 262144
for (HttpMessageReader<?> messageReader : exchangeStrategies.messageReaders()) {
if (messageReader instanceof DecoderHttpMessageReader) {
DecoderHttpMessageReader reader = (DecoderHttpMessageReader) messageReader;
Decoder decoder = reader.getDecoder();
if (decoder instanceof AbstractDataBufferDecoder) {
AbstractDataBufferDecoder dataBufferDecoder = (AbstractDataBufferDecoder)decoder;
dataBufferDecoder.setMaxInMemorySize(maxInMemorySize);
}
}
}
return exchangeStrategies;
}
@Override @Override
public int getOrder() { public int getOrder() {
return NettyWriteResponseFilter.WRITE_RESPONSE_FILTER_ORDER - 1; return NettyWriteResponseFilter.WRITE_RESPONSE_FILTER_ORDER - 1;

@ -13,6 +13,7 @@ import org.springframework.util.ClassUtils;
import org.springframework.util.ReflectionUtils; import org.springframework.util.ReflectionUtils;
import org.springframework.web.bind.support.WebDataBinderFactory; import org.springframework.web.bind.support.WebDataBinderFactory;
import org.springframework.web.context.request.NativeWebRequest; import org.springframework.web.context.request.NativeWebRequest;
import org.springframework.web.context.request.ServletWebRequest;
import org.springframework.web.context.request.WebRequest; import org.springframework.web.context.request.WebRequest;
import org.springframework.web.method.support.HandlerMethodArgumentResolver; import org.springframework.web.method.support.HandlerMethodArgumentResolver;
import org.springframework.web.method.support.ModelAndViewContainer; import org.springframework.web.method.support.ModelAndViewContainer;
@ -25,6 +26,7 @@ import org.springframework.web.servlet.mvc.method.annotation.ServletRequestMetho
import javax.servlet.ServletRequest; import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse; import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession; import javax.servlet.http.HttpSession;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
@ -117,6 +119,10 @@ public class ApiArgumentResolver implements SopHandlerMethodArgumentResolver {
, NativeWebRequest nativeWebRequest , NativeWebRequest nativeWebRequest
, WebDataBinderFactory webDataBinderFactory , WebDataBinderFactory webDataBinderFactory
) throws Exception { ) throws Exception {
nativeWebRequest = new SopServletWebRequest(
(HttpServletRequest) nativeWebRequest.getNativeRequest(),
(HttpServletResponse) nativeWebRequest.getNativeResponse()
);
if (openApiParams.contains(methodParameter)) { if (openApiParams.contains(methodParameter)) {
Object paramObj = this.getParamObject(methodParameter, nativeWebRequest); Object paramObj = this.getParamObject(methodParameter, nativeWebRequest);
if (paramObj != null) { if (paramObj != null) {
@ -175,10 +181,13 @@ public class ApiArgumentResolver implements SopHandlerMethodArgumentResolver {
* @param httpServletRequest * @param httpServletRequest
*/ */
protected void bindUploadFile(Object param, HttpServletRequest httpServletRequest) { protected void bindUploadFile(Object param, HttpServletRequest httpServletRequest) {
if (httpServletRequest instanceof MyServletRequestWrapper) {
httpServletRequest = (HttpServletRequest)((MyServletRequestWrapper) httpServletRequest).getRequest();
}
if (param == null) { if (param == null) {
return; return;
} }
if (this.isMultipartRequest(httpServletRequest)) { if (OpenUtil.isMultipart(httpServletRequest)) {
MultipartHttpServletRequest request = (MultipartHttpServletRequest) httpServletRequest; MultipartHttpServletRequest request = (MultipartHttpServletRequest) httpServletRequest;
Class<?> bizClass = param.getClass(); Class<?> bizClass = param.getClass();
ReflectionUtils.doWithFields(bizClass, field -> { ReflectionUtils.doWithFields(bizClass, field -> {
@ -190,10 +199,6 @@ public class ApiArgumentResolver implements SopHandlerMethodArgumentResolver {
} }
} }
protected boolean isMultipartRequest(HttpServletRequest request) {
return request instanceof MultipartRequest;
}
/** /**
* 获取其它的参数解析器 * 获取其它的参数解析器
* *
@ -221,4 +226,10 @@ public class ApiArgumentResolver implements SopHandlerMethodArgumentResolver {
public void setParamValidator(ParamValidator paramValidator) { public void setParamValidator(ParamValidator paramValidator) {
this.paramValidator = paramValidator; this.paramValidator = paramValidator;
} }
private static final class SopServletWebRequest extends ServletWebRequest {
public SopServletWebRequest(HttpServletRequest request, HttpServletResponse response) {
super(new MyServletRequestWrapper(request), response);
}
}
} }

@ -0,0 +1,53 @@
package com.gitee.sop.servercommon.param;
import javax.servlet.ReadListener;
import javax.servlet.ServletInputStream;
import java.io.IOException;
/**
* byte数组流包装
*
* @author tanghc
*/
public class ByteArrayStreamWrapper extends ServletInputStream {
private byte[] data;
private int idx = 0;
/**
* Creates a new <code>ByteArrayStreamWrapper</code> instance.
*
* @param data a <code>byte[]</code> value
*/
public ByteArrayStreamWrapper(byte[] data) {
if (data == null) {
data = new byte[0];
}
this.data = data;
}
@Override
public int read() throws IOException {
if (idx == data.length) {
return -1;
}
// I have to AND the byte with 0xff in order to ensure that it is returned as an unsigned integer
// the lack of this was causing a weird bug when manually unzipping gzipped request bodies
return data[idx++] & 0xff;
}
@Override
public boolean isFinished() {
return true;
}
@Override
public boolean isReady() {
return true;
}
@Override
public void setReadListener(ReadListener readListener) {
}
}

@ -0,0 +1,106 @@
package com.gitee.sop.servercommon.param;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import javax.servlet.ReadListener;
import javax.servlet.ServletInputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletRequestWrapper;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Map;
/**
* 包装HttpServletRequest
*/
@Slf4j
public class MyServletRequestWrapper extends HttpServletRequestWrapper {
private final byte[] body;
private final Map<String, String[]> parameterMap;
public MyServletRequestWrapper(HttpServletRequest request) {
super(request);
// 先缓存请求参数,不然获取不到,原因:
// 调用getInputStream方法时,usingInputStream会变成true
// 之后再调用request.getParameterMap(),就一直返回空map
// 因此要先调用request.getParameterMap()缓存起来
// 详见org.apache.catalina.connector.Request.parseParameters()中
// if (usingInputStream || usingReader) {
// success = true;
// return;
// }
parameterMap = request.getParameterMap();
try {
body = IOUtils.toByteArray(super.getInputStream());
} catch (IOException e) {
log.error("cache body error, url:{}",request.getRequestURI(), e);
throw new RuntimeException("cache body error", e);
}
}
@Override
public Map<String, String[]> getParameterMap() {
return parameterMap;
}
@Override
public BufferedReader getReader() throws IOException {
return new BufferedReader(new InputStreamReader(getInputStream()));
}
@Override
public ServletInputStream getInputStream() throws IOException {
return new ByteArrayStreamWrapper(body);
}
/**
* byte数组流包装
*
* @author tanghc
*/
public static class ByteArrayStreamWrapper extends ServletInputStream {
private byte[] data;
private int idx = 0;
/**
* Creates a new <code>ByteArrayStreamWrapper</code> instance.
*
* @param data a <code>byte[]</code> value
*/
public ByteArrayStreamWrapper(byte[] data) {
if (data == null) {
data = new byte[0];
}
this.data = data;
}
@Override
public int read() throws IOException {
if (idx == data.length) {
return -1;
}
// I have to AND the byte with 0xff in order to ensure that it is returned as an unsigned integer
// the lack of this was causing a weird bug when manually unzipping gzipped request bodies
return data[idx++] & 0xff;
}
@Override
public boolean isFinished() {
return true;
}
@Override
public boolean isReady() {
return true;
}
@Override
public void setReadListener(ReadListener readListener) {
}
}
}

@ -14,6 +14,7 @@ import java.net.URLDecoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -23,6 +24,8 @@ import java.util.Set;
@Slf4j @Slf4j
public class OpenUtil { public class OpenUtil {
public static final String MULTIPART = "multipart/";
/** /**
* 将get类型的参数转换成map * 将get类型的参数转换成map
* *
@ -120,4 +123,19 @@ public class OpenUtil {
return serverSign.equals(sign); return serverSign.equals(sign);
} }
/**
* 是否是文件上传请求
*
* @param request 请求
* @return true
*/
public static boolean isMultipart(HttpServletRequest request) {
String contentType = request.getContentType();
// Don't use this filter on GET method
if (contentType == null) {
return false;
}
return contentType.toLowerCase(Locale.ENGLISH).startsWith(MULTIPART);
}
} }

@ -56,6 +56,11 @@
<artifactId>swagger-bootstrap-ui</artifactId> <artifactId>swagger-bootstrap-ui</artifactId>
<version>1.9.5</version> <version>1.9.5</version>
</dependency> </dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>2.9.2</version>
</dependency>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>

@ -1,5 +1,7 @@
package com.gitee.sop.storyweb.controller; package com.gitee.sop.storyweb.controller;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.gitee.sop.servercommon.annotation.BizCode; import com.gitee.sop.servercommon.annotation.BizCode;
import com.gitee.sop.servercommon.annotation.Open; import com.gitee.sop.servercommon.annotation.Open;
import com.gitee.sop.storyweb.controller.param.CategoryParam; import com.gitee.sop.storyweb.controller.param.CategoryParam;
@ -12,13 +14,18 @@ import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date; import java.util.Date;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* 签名验证通过后到达这里进行具体的业务处理 * 签名验证通过后到达这里进行具体的业务处理
@ -83,9 +90,10 @@ public class Example1001_BaseController {
// 忽略验证 // 忽略验证
@ApiOperation(value = "忽略签名验证", notes = "忽略签名验证") @ApiOperation(value = "忽略签名验证", notes = "忽略签名验证")
@Open(value = "story.get.ignore", ignoreValidate = true) @Open(value = "story.get.ignore", ignoreValidate = true)
@RequestMapping("/get/ignore/v1") @RequestMapping(value = "/get/ignore/v1", method = {RequestMethod.GET, RequestMethod.POST})
public StoryResult getStory21(StoryParam story) { public StoryResult getStory21(@RequestBody StoryParam story) {
StoryResult result = new StoryResult(); StoryResult result = new StoryResult();
result.setId((long) story.getId());
result.setName(story.getName() + ", ignoreValidate = true"); result.setName(story.getName() + ", ignoreValidate = true");
return result; return result;
} }
@ -160,4 +168,19 @@ public class Example1001_BaseController {
return parent; return parent;
} }
private static String json = "{\"flightDate\":\"2020-09-01\",\"flightNo\":\"HO1705\",\"departureAirport\":\"ZSCN\",\"arrivalAirport\":\"ZPLJ\",\"ycy\":\"11521\",\"lcy\":\"4354\",\"cr\":\"145\",\"et\":\"1\",\"ye\":\"0\",\"td\":\"0\",\"gw\":\"0\",\"ew\":\"146\",\"xl\":\"1018\",\"yj\":\"0\",\"hw\":\"635\"}";
// 返回大数据
@Open(value = "bigdata.get")
@RequestMapping("/bigdata/v1")
public Map<String, Object> bigData(StoryParam param) {
int len = 2000;
List<JSONObject> list = new ArrayList<>(len);
for (int i = 0; i < len; i++) {
list.add(JSON.parseObject(json));
}
Map<String, Object> map = new HashMap<>();
map.put("data", list);
return map;
}
} }

@ -8,4 +8,8 @@ mysql.password=root
# nacos注册中心地址 # nacos注册中心地址
register.url=http://localhost:1111/eureka/ register.url=http://localhost:1111/eureka/
# POST请求最大缓存,默认是262144(256KB),即:256 * 1024
# 如果需要传输大文本可以把值设置高一点
spring.codec.max-in-memory-size=262144
logging.level.com.gitee=debug logging.level.com.gitee=debug

@ -275,6 +275,25 @@ public class AllInOneTest extends TestBase {
client.execute(requestBuilder); client.execute(requestBuilder);
} }
/**
* 测试返回大json数据
*/
public void testLargeResponseJson() {
Client.RequestBuilder requestBuilder = new Client.RequestBuilder()
.method("bigdata.get")
.version("1.0")
.bizContent(new BizContent().add("id", "1").add("name", "葫芦娃"))
.httpMethod(HttpTool.HTTPMethod.GET)
.callback((requestInfo, responseData) -> {
int size = JSON.parseObject(responseData)
.getJSONObject("bigdata_get_response")
.getJSONArray("data").size();
Assert.assertEquals(size, 2000);
});
client.execute(requestBuilder);
}
static class BizContent extends HashMap<String, Object> { static class BizContent extends HashMap<String, Object> {
public BizContent add(String key, Object value) { public BizContent add(String key, Object value) {
this.put(key, value); this.put(key, value);

Loading…
Cancel
Save