Merge branch 'master' into eureka

changelog.md

@@ -4,6 +4,7 @@
 
 - 可定义业务错误码（见`@Open`注解中的`bizCode`属性）
 - 文档参数可指定最大长度（使用`@Length(max = xx)`）
+- 修复返回大文本导致的错误（Exceeded limit on max bytes to buffer : 262144）
 
 ## 4.0.2
 

sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/GatewayModifyResponseGatewayFilter.java

@@ -5,6 +5,7 @@ import com.gitee.sop.gatewaycommon.bean.SopConstants;
 import com.gitee.sop.gatewaycommon.result.ResultExecutor;
 import org.apache.commons.lang3.StringUtils;
 import org.reactivestreams.Publisher;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.cloud.gateway.filter.GatewayFilterChain;
 import org.springframework.cloud.gateway.filter.GlobalFilter;
 import org.springframework.cloud.gateway.filter.NettyWriteResponseFilter;
@@ -12,12 +13,16 @@ import org.springframework.cloud.gateway.filter.factory.rewrite.CachedBodyOutput
 import org.springframework.cloud.gateway.support.BodyInserterContext;
 import org.springframework.cloud.gateway.support.DefaultClientResponse;
 import org.springframework.core.Ordered;
+import org.springframework.core.codec.AbstractDataBufferDecoder;
+import org.springframework.core.codec.Decoder;
 import org.springframework.core.io.buffer.DataBuffer;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseCookie;
 import org.springframework.http.client.reactive.ClientHttpResponse;
+import org.springframework.http.codec.DecoderHttpMessageReader;
+import org.springframework.http.codec.HttpMessageReader;
 import org.springframework.http.server.reactive.ServerHttpResponseDecorator;
 import org.springframework.util.MultiValueMap;
 import org.springframework.web.reactive.function.BodyInserter;
@@ -35,6 +40,8 @@ import static org.springframework.cloud.gateway.support.ServerWebExchangeUtils.O
  */
 public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered {
 
+    @Value("${spring.codec.max-in-memory-size:262144}")
+    private int maxInMemorySize;
 
     @Override
     @SuppressWarnings("unchecked")
@@ -60,7 +67,7 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered
                 //this will prevent exception in case of using non-standard media types like "Content-Type: image"
                 httpHeaders.add(HttpHeaders.CONTENT_TYPE, originalResponseContentType);
                 ResponseAdapter responseAdapter = new ResponseAdapter(body, httpHeaders);
-                DefaultClientResponse clientResponse = new DefaultClientResponse(responseAdapter, ExchangeStrategies.withDefaults());
+                DefaultClientResponse clientResponse = new DefaultClientResponse(responseAdapter, getExchangeStrategies());
 
                 //TODO: flux or mono
                 Mono modifiedBody = clientResponse.bodyToMono(inClass)
@@ -97,6 +104,22 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered
         return chain.filter(exchange.mutate().response(responseDecorator).build());
     }
 
+    private ExchangeStrategies getExchangeStrategies() {
+        ExchangeStrategies exchangeStrategies = ExchangeStrategies.withDefaults();
+        // 修复返回大文本数据报org.springframework.core.io.buffer.DataBufferLimitException: Exceeded limit on max bytes to buffer : 262144
+        for (HttpMessageReader<?> messageReader : exchangeStrategies.messageReaders()) {
+            if (messageReader instanceof DecoderHttpMessageReader) {
+                DecoderHttpMessageReader reader = (DecoderHttpMessageReader) messageReader;
+                Decoder decoder = reader.getDecoder();
+                if (decoder instanceof AbstractDataBufferDecoder) {
+                    AbstractDataBufferDecoder dataBufferDecoder = (AbstractDataBufferDecoder)decoder;
+                    dataBufferDecoder.setMaxInMemorySize(maxInMemorySize);
+                }
+            }
+        }
+        return exchangeStrategies;
+    }
+
     @Override
     public int getOrder() {
         return NettyWriteResponseFilter.WRITE_RESPONSE_FILTER_ORDER - 1;

sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/ApiArgumentResolver.java

@@ -13,6 +13,7 @@ import org.springframework.util.ClassUtils;
 import org.springframework.util.ReflectionUtils;
 import org.springframework.web.bind.support.WebDataBinderFactory;
 import org.springframework.web.context.request.NativeWebRequest;
+import org.springframework.web.context.request.ServletWebRequest;
 import org.springframework.web.context.request.WebRequest;
 import org.springframework.web.method.support.HandlerMethodArgumentResolver;
 import org.springframework.web.method.support.ModelAndViewContainer;
@@ -25,6 +26,7 @@ import org.springframework.web.servlet.mvc.method.annotation.ServletRequestMetho
 import javax.servlet.ServletRequest;
 import javax.servlet.ServletResponse;
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.HttpSession;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -117,6 +119,10 @@ public class ApiArgumentResolver implements SopHandlerMethodArgumentResolver {
             , NativeWebRequest nativeWebRequest
             , WebDataBinderFactory webDataBinderFactory
     ) throws Exception {
+        nativeWebRequest = new SopServletWebRequest(
+                (HttpServletRequest) nativeWebRequest.getNativeRequest(),
+                (HttpServletResponse) nativeWebRequest.getNativeResponse()
+        );
         if (openApiParams.contains(methodParameter)) {
             Object paramObj = this.getParamObject(methodParameter, nativeWebRequest);
             if (paramObj != null) {
@@ -175,10 +181,13 @@ public class ApiArgumentResolver implements SopHandlerMethodArgumentResolver {
      * @param httpServletRequest
      */
     protected void bindUploadFile(Object param, HttpServletRequest httpServletRequest) {
+        if (httpServletRequest instanceof MyServletRequestWrapper) {
+            httpServletRequest = (HttpServletRequest)((MyServletRequestWrapper) httpServletRequest).getRequest();
+        }
         if (param == null) {
             return;
         }
-        if (this.isMultipartRequest(httpServletRequest)) {
+        if (OpenUtil.isMultipart(httpServletRequest)) {
             MultipartHttpServletRequest request = (MultipartHttpServletRequest) httpServletRequest;
             Class<?> bizClass = param.getClass();
             ReflectionUtils.doWithFields(bizClass, field -> {
@@ -190,10 +199,6 @@ public class ApiArgumentResolver implements SopHandlerMethodArgumentResolver {
         }
     }
 
-    protected boolean isMultipartRequest(HttpServletRequest request) {
-        return request instanceof MultipartRequest;
-    }
-
     /**
      * 获取其它的参数解析器
      *
@@ -221,4 +226,10 @@ public class ApiArgumentResolver implements SopHandlerMethodArgumentResolver {
     public void setParamValidator(ParamValidator paramValidator) {
         this.paramValidator = paramValidator;
     }
+
+    private static final class SopServletWebRequest extends ServletWebRequest {
+        public SopServletWebRequest(HttpServletRequest request, HttpServletResponse response) {
+            super(new MyServletRequestWrapper(request), response);
+        }
+    }
 }

sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/ByteArrayStreamWrapper.java

@@ -0,0 +1,53 @@
+package com.gitee.sop.servercommon.param;
+
+import javax.servlet.ReadListener;
+import javax.servlet.ServletInputStream;
+import java.io.IOException;
+
+/**
+ * byte数组流包装
+ *
+ * @author tanghc
+ */
+public class ByteArrayStreamWrapper extends ServletInputStream {
+
+    private byte[] data;
+    private int idx = 0;
+
+    /**
+     * Creates a new <code>ByteArrayStreamWrapper</code> instance.
+     *
+     * @param data a <code>byte[]</code> value
+     */
+    public ByteArrayStreamWrapper(byte[] data) {
+        if (data == null) {
+            data = new byte[0];
+        }
+        this.data = data;
+    }
+
+    @Override
+    public int read() throws IOException {
+        if (idx == data.length) {
+            return -1;
+        }
+        // I have to AND the byte with 0xff in order to ensure that it is returned as an unsigned integer
+        // the lack of this was causing a weird bug when manually unzipping gzipped request bodies
+        return data[idx++] & 0xff;
+    }
+
+    @Override
+    public boolean isFinished() {
+        return true;
+    }
+
+    @Override
+    public boolean isReady() {
+        return true;
+    }
+
+    @Override
+    public void setReadListener(ReadListener readListener) {
+
+    }
+}

sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/param/MyServletRequestWrapper.java

@@ -0,0 +1,106 @@
+package com.gitee.sop.servercommon.param;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+
+import javax.servlet.ReadListener;
+import javax.servlet.ServletInputStream;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Map;
+
+/**
+ * 包装HttpServletRequest
+ */
+@Slf4j
+public class MyServletRequestWrapper extends HttpServletRequestWrapper {
+
+    private final byte[] body;
+    private final Map<String, String[]> parameterMap;
+
+    public MyServletRequestWrapper(HttpServletRequest request) {
+        super(request);
+        // 先缓存请求参数，不然获取不到，原因：
+        // 调用getInputStream方法时，usingInputStream会变成true
+        // 之后再调用request.getParameterMap()，就一直返回空map
+        // 因此要先调用request.getParameterMap()缓存起来
+        // 详见org.apache.catalina.connector.Request.parseParameters()中
+        // if (usingInputStream || usingReader) {
+        //                success = true;
+        //                return;
+        //  }
+        parameterMap = request.getParameterMap();
+        try {
+            body = IOUtils.toByteArray(super.getInputStream());
+        } catch (IOException e) {
+            log.error("cache body error, url:{}",request.getRequestURI(), e);
+            throw new RuntimeException("cache body error", e);
+        }
+    }
+
+    @Override
+    public Map<String, String[]> getParameterMap() {
+        return parameterMap;
+    }
+
+    @Override
+    public BufferedReader getReader() throws IOException {
+        return new BufferedReader(new InputStreamReader(getInputStream()));
+    }
+
+    @Override
+    public ServletInputStream getInputStream() throws IOException {
+        return new ByteArrayStreamWrapper(body);
+    }
+
+    /**
+     * byte数组流包装
+     *
+     * @author tanghc
+     */
+    public static class ByteArrayStreamWrapper extends ServletInputStream {
+
+        private byte[] data;
+        private int idx = 0;
+
+        /**
+         * Creates a new <code>ByteArrayStreamWrapper</code> instance.
+         *
+         * @param data a <code>byte[]</code> value
+         */
+        public ByteArrayStreamWrapper(byte[] data) {
+            if (data == null) {
+                data = new byte[0];
+            }
+            this.data = data;
+        }
+
+        @Override
+        public int read() throws IOException {
+            if (idx == data.length) {
+                return -1;
+            }
+            // I have to AND the byte with 0xff in order to ensure that it is returned as an unsigned integer
+            // the lack of this was causing a weird bug when manually unzipping gzipped request bodies
+            return data[idx++] & 0xff;
+        }
+
+        @Override
+        public boolean isFinished() {
+            return true;
+        }
+
+        @Override
+        public boolean isReady() {
+            return true;
+        }
+
+        @Override
+        public void setReadListener(ReadListener readListener) {
+
+        }
+    }
+}

sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/util/OpenUtil.java

@@ -14,6 +14,7 @@ import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
@@ -23,6 +24,8 @@ import java.util.Set;
 @Slf4j
 public class OpenUtil {
 
+    public static final String MULTIPART = "multipart/";
+
     /**
      * 将get类型的参数转换成map，
      *
@@ -120,4 +123,19 @@ public class OpenUtil {
         return serverSign.equals(sign);
     }
 
+    /**
+     * 是否是文件上传请求
+     *
+     * @param request 请求
+     * @return true：是
+     */
+    public static boolean isMultipart(HttpServletRequest request) {
+        String contentType = request.getContentType();
+        // Don't use this filter on GET method
+        if (contentType == null) {
+            return false;
+        }
+        return contentType.toLowerCase(Locale.ENGLISH).startsWith(MULTIPART);
+    }
+
 }

sop-example/sop-story/pom.xml

@@ -56,6 +56,11 @@
             <artifactId>swagger-bootstrap-ui</artifactId>
             <version>1.9.5</version>
         </dependency>
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-swagger-ui</artifactId>
+            <version>2.9.2</version>
+        </dependency>
 
         <dependency>
             <groupId>org.springframework.boot</groupId>

sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java

@@ -1,5 +1,7 @@
 package com.gitee.sop.storyweb.controller;
 
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
 import com.gitee.sop.servercommon.annotation.BizCode;
 import com.gitee.sop.servercommon.annotation.Open;
 import com.gitee.sop.storyweb.controller.param.CategoryParam;
@@ -12,13 +14,18 @@ import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RestController;
 
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * 签名验证通过后，到达这里进行具体的业务处理。
@@ -83,9 +90,10 @@ public class Example1001_BaseController {
     // 忽略验证
     @ApiOperation(value = "忽略签名验证", notes = "忽略签名验证")
     @Open(value = "story.get.ignore", ignoreValidate = true)
-    @RequestMapping("/get/ignore/v1")
+    @RequestMapping(value = "/get/ignore/v1", method = {RequestMethod.GET, RequestMethod.POST})
-    public StoryResult getStory21(StoryParam story) {
+    public StoryResult getStory21(@RequestBody StoryParam story) {
         StoryResult result = new StoryResult();
+        result.setId((long) story.getId());
         result.setName(story.getName() + ", ignoreValidate = true");
         return result;
     }
@@ -160,4 +168,19 @@ public class Example1001_BaseController {
         return parent;
     }
 
+    private static String json = "{\"flightDate\":\"2020-09-01\",\"flightNo\":\"HO1705\",\"departureAirport\":\"ZSCN\",\"arrivalAirport\":\"ZPLJ\",\"ycy\":\"11521\",\"lcy\":\"4354\",\"cr\":\"145\",\"et\":\"1\",\"ye\":\"0\",\"td\":\"0\",\"gw\":\"0\",\"ew\":\"146\",\"xl\":\"1018\",\"yj\":\"0\",\"hw\":\"635\"}";
+    // 返回大数据
+    @Open(value = "bigdata.get")
+    @RequestMapping("/bigdata/v1")
+    public Map<String, Object> bigData(StoryParam param) {
+        int len = 2000;
+        List<JSONObject> list = new ArrayList<>(len);
+        for (int i = 0; i < len; i++) {
+            list.add(JSON.parseObject(json));
+        }
+        Map<String, Object> map = new HashMap<>();
+        map.put("data", list);
+        return map;
+    }
+
 }

sop-gateway/src/main/resources/application-dev.properties

@@ -8,4 +8,8 @@ mysql.password=root
 # nacos注册中心地址
 register.url=http://localhost:1111/eureka/
 
+# POST请求最大缓存，默认是262144（256KB），即：256 * 1024
+# 如果需要传输大文本可以把值设置高一点
+spring.codec.max-in-memory-size=262144
+
 logging.level.com.gitee=debug

sop-test/src/test/java/com/gitee/sop/test/AllInOneTest.java

@@ -275,6 +275,25 @@ public class AllInOneTest extends TestBase {
         client.execute(requestBuilder);
     }
 
+    /**
+     * 测试返回大json数据
+     */
+    public void testLargeResponseJson() {
+        Client.RequestBuilder requestBuilder = new Client.RequestBuilder()
+                .method("bigdata.get")
+                .version("1.0")
+                .bizContent(new BizContent().add("id", "1").add("name", "葫芦娃"))
+                .httpMethod(HttpTool.HTTPMethod.GET)
+                .callback((requestInfo, responseData) -> {
+                    int size = JSON.parseObject(responseData)
+                            .getJSONObject("bigdata_get_response")
+                            .getJSONArray("data").size();
+                    Assert.assertEquals(size, 2000);
+                });
+
+        client.execute(requestBuilder);
+    }
+
     static class BizContent extends HashMap<String, Object> {
         public BizContent add(String key, Object value) {
             this.put(key, value);