diff --git a/changelog.md b/changelog.md index 2fb8a76c..551ac68e 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,7 @@ - 可定义业务错误码（见`@Open`注解中的`bizCode`属性） - 文档参数可指定最大长度（使用`@Length(max = xx)`） +- 修复返回大文本导致的错误（Exceeded limit on max bytes to buffer : 262144） ## 4.0.2 diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/GatewayModifyResponseGatewayFilter.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/GatewayModifyResponseGatewayFilter.java index ca72a59c..e6a13901 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/GatewayModifyResponseGatewayFilter.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/GatewayModifyResponseGatewayFilter.java @@ -5,6 +5,7 @@ import com.gitee.sop.gatewaycommon.bean.SopConstants; import com.gitee.sop.gatewaycommon.result.ResultExecutor; import org.apache.commons.lang3.StringUtils; import org.reactivestreams.Publisher; +import org.springframework.beans.factory.annotation.Value; import org.springframework.cloud.gateway.filter.GatewayFilterChain; import org.springframework.cloud.gateway.filter.GlobalFilter; import org.springframework.cloud.gateway.filter.NettyWriteResponseFilter; @@ -12,12 +13,16 @@ import org.springframework.cloud.gateway.filter.factory.rewrite.CachedBodyOutput import org.springframework.cloud.gateway.support.BodyInserterContext; import org.springframework.cloud.gateway.support.DefaultClientResponse; import org.springframework.core.Ordered; +import org.springframework.core.codec.AbstractDataBufferDecoder; +import org.springframework.core.codec.Decoder; import org.springframework.core.io.buffer.DataBuffer; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseCookie; import org.springframework.http.client.reactive.ClientHttpResponse; +import org.springframework.http.codec.DecoderHttpMessageReader; +import org.springframework.http.codec.HttpMessageReader; import org.springframework.http.server.reactive.ServerHttpResponseDecorator; import org.springframework.util.MultiValueMap; import org.springframework.web.reactive.function.BodyInserter; @@ -35,6 +40,8 @@ import static org.springframework.cloud.gateway.support.ServerWebExchangeUtils.O */ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered { + @Value("${spring.codec.max-in-memory-size:262144}") + private int maxInMemorySize; @Override @SuppressWarnings("unchecked") @@ -60,7 +67,7 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered //this will prevent exception in case of using non-standard media types like "Content-Type: image" httpHeaders.add(HttpHeaders.CONTENT_TYPE, originalResponseContentType); ResponseAdapter responseAdapter = new ResponseAdapter(body, httpHeaders); - DefaultClientResponse clientResponse = new DefaultClientResponse(responseAdapter, ExchangeStrategies.withDefaults()); + DefaultClientResponse clientResponse = new DefaultClientResponse(responseAdapter, getExchangeStrategies()); //TODO: flux or mono Mono modifiedBody = clientResponse.bodyToMono(inClass) @@ -97,6 +104,22 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered return chain.filter(exchange.mutate().response(responseDecorator).build()); } + private ExchangeStrategies getExchangeStrategies() { + ExchangeStrategies exchangeStrategies = ExchangeStrategies.withDefaults(); + // 修复返回大文本数据报org.springframework.core.io.buffer.DataBufferLimitException: Exceeded limit on max bytes to buffer : 262144 + for (HttpMessageReader messageReader : exchangeStrategies.messageReaders()) { + if (messageReader instanceof DecoderHttpMessageReader) { + DecoderHttpMessageReader reader = (DecoderHttpMessageReader) messageReader; + Decoder decoder = reader.getDecoder(); + if (decoder instanceof AbstractDataBufferDecoder) { + AbstractDataBufferDecoder dataBufferDecoder = (AbstractDataBufferDecoder)decoder; + dataBufferDecoder.setMaxInMemorySize(maxInMemorySize); + } + } + } + return exchangeStrategies; + } + @Override public int getOrder() { return NettyWriteResponseFilter.WRITE_RESPONSE_FILTER_ORDER - 1; diff --git a/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/ApiArgumentResolver.java b/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/ApiArgumentResolver.java index 4e94e389..71052dbd 100644 --- a/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/ApiArgumentResolver.java +++ b/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/ApiArgumentResolver.java @@ -13,6 +13,7 @@ import org.springframework.util.ClassUtils; import org.springframework.util.ReflectionUtils; import org.springframework.web.bind.support.WebDataBinderFactory; import org.springframework.web.context.request.NativeWebRequest; +import org.springframework.web.context.request.ServletWebRequest; import org.springframework.web.context.request.WebRequest; import org.springframework.web.method.support.HandlerMethodArgumentResolver; import org.springframework.web.method.support.ModelAndViewContainer; @@ -25,6 +26,7 @@ import org.springframework.web.servlet.mvc.method.annotation.ServletRequestMetho import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; import java.io.InputStream; import java.io.OutputStream; @@ -117,6 +119,10 @@ public class ApiArgumentResolver implements SopHandlerMethodArgumentResolver { , NativeWebRequest nativeWebRequest , WebDataBinderFactory webDataBinderFactory ) throws Exception { + nativeWebRequest = new SopServletWebRequest( + (HttpServletRequest) nativeWebRequest.getNativeRequest(), + (HttpServletResponse) nativeWebRequest.getNativeResponse() + ); if (openApiParams.contains(methodParameter)) { Object paramObj = this.getParamObject(methodParameter, nativeWebRequest); if (paramObj != null) { @@ -175,10 +181,13 @@ public class ApiArgumentResolver implements SopHandlerMethodArgumentResolver { * @param httpServletRequest */ protected void bindUploadFile(Object param, HttpServletRequest httpServletRequest) { + if (httpServletRequest instanceof MyServletRequestWrapper) { + httpServletRequest = (HttpServletRequest)((MyServletRequestWrapper) httpServletRequest).getRequest(); + } if (param == null) { return; } - if (this.isMultipartRequest(httpServletRequest)) { + if (OpenUtil.isMultipart(httpServletRequest)) { MultipartHttpServletRequest request = (MultipartHttpServletRequest) httpServletRequest; Class bizClass = param.getClass(); ReflectionUtils.doWithFields(bizClass, field -> { @@ -190,10 +199,6 @@ public class ApiArgumentResolver implements SopHandlerMethodArgumentResolver { } } - protected boolean isMultipartRequest(HttpServletRequest request) { - return request instanceof MultipartRequest; - } - /** * 获取其它的参数解析器 * @@ -221,4 +226,10 @@ public class ApiArgumentResolver implements SopHandlerMethodArgumentResolver { public void setParamValidator(ParamValidator paramValidator) { this.paramValidator = paramValidator; } + + private static final class SopServletWebRequest extends ServletWebRequest { + public SopServletWebRequest(HttpServletRequest request, HttpServletResponse response) { + super(new MyServletRequestWrapper(request), response); + } + } } \ No newline at end of file diff --git a/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/ByteArrayStreamWrapper.java b/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/ByteArrayStreamWrapper.java new file mode 100644 index 00000000..64ede5fa --- /dev/null +++ b/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/ByteArrayStreamWrapper.java @@ -0,0 +1,53 @@ +package com.gitee.sop.servercommon.param; + +import javax.servlet.ReadListener; +import javax.servlet.ServletInputStream; +import java.io.IOException; + +/** + * byte数组流包装 + * + * @author tanghc + */ +public class ByteArrayStreamWrapper extends ServletInputStream { + + private byte[] data; + private int idx = 0; + + /** + * Creates a new ByteArrayStreamWrapper instance. + * + * @param data a byte[] value + */ + public ByteArrayStreamWrapper(byte[] data) { + if (data == null) { + data = new byte[0]; + } + this.data = data; + } + + @Override + public int read() throws IOException { + if (idx == data.length) { + return -1; + } + // I have to AND the byte with 0xff in order to ensure that it is returned as an unsigned integer + // the lack of this was causing a weird bug when manually unzipping gzipped request bodies + return data[idx++] & 0xff; + } + + @Override + public boolean isFinished() { + return true; + } + + @Override + public boolean isReady() { + return true; + } + + @Override + public void setReadListener(ReadListener readListener) { + + } +} \ No newline at end of file diff --git a/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/MyServletRequestWrapper.java b/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/MyServletRequestWrapper.java new file mode 100644 index 00000000..0c14c6eb --- /dev/null +++ b/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/MyServletRequestWrapper.java @@ -0,0 +1,106 @@ +package com.gitee.sop.servercommon.param; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; + +import javax.servlet.ReadListener; +import javax.servlet.ServletInputStream; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Map; + +/** + * 包装HttpServletRequest + */ +@Slf4j +public class MyServletRequestWrapper extends HttpServletRequestWrapper { + + private final byte[] body; + private final Map parameterMap; + + public MyServletRequestWrapper(HttpServletRequest request) { + super(request); + // 先缓存请求参数，不然获取不到，原因： + // 调用getInputStream方法时，usingInputStream会变成true + // 之后再调用request.getParameterMap()，就一直返回空map + // 因此要先调用request.getParameterMap()缓存起来 + // 详见org.apache.catalina.connector.Request.parseParameters()中 + // if (usingInputStream || usingReader) { + // success = true; + // return; + // } + parameterMap = request.getParameterMap(); + try { + body = IOUtils.toByteArray(super.getInputStream()); + } catch (IOException e) { + log.error("cache body error, url:{}",request.getRequestURI(), e); + throw new RuntimeException("cache body error", e); + } + } + + @Override + public Map getParameterMap() { + return parameterMap; + } + + @Override + public BufferedReader getReader() throws IOException { + return new BufferedReader(new InputStreamReader(getInputStream())); + } + + @Override + public ServletInputStream getInputStream() throws IOException { + return new ByteArrayStreamWrapper(body); + } + + /** + * byte数组流包装 + * + * @author tanghc + */ + public static class ByteArrayStreamWrapper extends ServletInputStream { + + private byte[] data; + private int idx = 0; + + /** + * Creates a new ByteArrayStreamWrapper instance. + * + * @param data a byte[] value + */ + public ByteArrayStreamWrapper(byte[] data) { + if (data == null) { + data = new byte[0]; + } + this.data = data; + } + + @Override + public int read() throws IOException { + if (idx == data.length) { + return -1; + } + // I have to AND the byte with 0xff in order to ensure that it is returned as an unsigned integer + // the lack of this was causing a weird bug when manually unzipping gzipped request bodies + return data[idx++] & 0xff; + } + + @Override + public boolean isFinished() { + return true; + } + + @Override + public boolean isReady() { + return true; + } + + @Override + public void setReadListener(ReadListener readListener) { + + } + } +} \ No newline at end of file diff --git a/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/util/OpenUtil.java b/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/util/OpenUtil.java index b048ea7b..727e5168 100644 --- a/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/util/OpenUtil.java +++ b/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/util/OpenUtil.java @@ -14,6 +14,7 @@ import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Set; @@ -23,6 +24,8 @@ import java.util.Set; @Slf4j public class OpenUtil { + public static final String MULTIPART = "multipart/"; + /** * 将get类型的参数转换成map， * @@ -120,4 +123,19 @@ public class OpenUtil { return serverSign.equals(sign); } + /** + * 是否是文件上传请求 + * + * @param request 请求 + * @return true：是 + */ + public static boolean isMultipart(HttpServletRequest request) { + String contentType = request.getContentType(); + // Don't use this filter on GET method + if (contentType == null) { + return false; + } + return contentType.toLowerCase(Locale.ENGLISH).startsWith(MULTIPART); + } + } diff --git a/sop-example/sop-story/pom.xml b/sop-example/sop-story/pom.xml index f45edd17..f7cd0ca0 100644 --- a/sop-example/sop-story/pom.xml +++ b/sop-example/sop-story/pom.xml @@ -56,6 +56,11 @@ swagger-bootstrap-ui 1.9.5 + + io.springfox + springfox-swagger-ui + 2.9.2 + org.springframework.boot diff --git a/sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java b/sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java index b2eb6e2c..65c18392 100644 --- a/sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java +++ b/sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java @@ -1,5 +1,7 @@ package com.gitee.sop.storyweb.controller; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; import com.gitee.sop.servercommon.annotation.BizCode; import com.gitee.sop.servercommon.annotation.Open; import com.gitee.sop.storyweb.controller.param.CategoryParam; @@ -12,13 +14,18 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * 签名验证通过后，到达这里进行具体的业务处理。 @@ -83,9 +90,10 @@ public class Example1001_BaseController { // 忽略验证 @ApiOperation(value = "忽略签名验证", notes = "忽略签名验证") @Open(value = "story.get.ignore", ignoreValidate = true) - @RequestMapping("/get/ignore/v1") - public StoryResult getStory21(StoryParam story) { + @RequestMapping(value = "/get/ignore/v1", method = {RequestMethod.GET, RequestMethod.POST}) + public StoryResult getStory21(@RequestBody StoryParam story) { StoryResult result = new StoryResult(); + result.setId((long) story.getId()); result.setName(story.getName() + ", ignoreValidate = true"); return result; } @@ -160,4 +168,19 @@ public class Example1001_BaseController { return parent; } + private static String json = "{\"flightDate\":\"2020-09-01\",\"flightNo\":\"HO1705\",\"departureAirport\":\"ZSCN\",\"arrivalAirport\":\"ZPLJ\",\"ycy\":\"11521\",\"lcy\":\"4354\",\"cr\":\"145\",\"et\":\"1\",\"ye\":\"0\",\"td\":\"0\",\"gw\":\"0\",\"ew\":\"146\",\"xl\":\"1018\",\"yj\":\"0\",\"hw\":\"635\"}"; + // 返回大数据 + @Open(value = "bigdata.get") + @RequestMapping("/bigdata/v1") + public Map bigData(StoryParam param) { + int len = 2000; + List list = new ArrayList<>(len); + for (int i = 0; i < len; i++) { + list.add(JSON.parseObject(json)); + } + Map map = new HashMap<>(); + map.put("data", list); + return map; + } + } diff --git a/sop-gateway/src/main/resources/application-dev.properties b/sop-gateway/src/main/resources/application-dev.properties index 4d02bb23..3ed92423 100644 --- a/sop-gateway/src/main/resources/application-dev.properties +++ b/sop-gateway/src/main/resources/application-dev.properties @@ -8,4 +8,8 @@ mysql.password=root # nacos注册中心地址 register.url=http://localhost:1111/eureka/ +# POST请求最大缓存，默认是262144（256KB），即：256 * 1024 +# 如果需要传输大文本可以把值设置高一点 +spring.codec.max-in-memory-size=262144 + logging.level.com.gitee=debug \ No newline at end of file diff --git a/sop-test/src/test/java/com/gitee/sop/test/AllInOneTest.java b/sop-test/src/test/java/com/gitee/sop/test/AllInOneTest.java index 3ac7f602..cae5bd1b 100644 --- a/sop-test/src/test/java/com/gitee/sop/test/AllInOneTest.java +++ b/sop-test/src/test/java/com/gitee/sop/test/AllInOneTest.java @@ -275,6 +275,25 @@ public class AllInOneTest extends TestBase { client.execute(requestBuilder); } + /** + * 测试返回大json数据 + */ + public void testLargeResponseJson() { + Client.RequestBuilder requestBuilder = new Client.RequestBuilder() + .method("bigdata.get") + .version("1.0") + .bizContent(new BizContent().add("id", "1").add("name", "葫芦娃")) + .httpMethod(HttpTool.HTTPMethod.GET) + .callback((requestInfo, responseData) -> { + int size = JSON.parseObject(responseData) + .getJSONObject("bigdata_get_response") + .getJSONArray("data").size(); + Assert.assertEquals(size, 2000); + }); + + client.execute(requestBuilder); + } + static class BizContent extends HashMap { public BizContent add(String key, Object value) { this.put(key, value);