From 2c765af0362c9a057a86a9b709cbca0b98eee445 Mon Sep 17 00:00:00 2001 From: tanghc Date: Mon, 1 Mar 2021 09:02:19 +0800 Subject: [PATCH 1/9] 4.2.7 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 62da1d0a..248fb9db 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,6 @@ System.out.println(responseData); ## 沟通交流 -Q群：167643071 +- QQ群 -![SOP2群](https://images.gitee.com/uploads/images/2020/0203/103119_bdf84eb6_332975.png "SOP2群.png") +1群：167643071（满）、2群：167643071（满）、3群：565894770 From 5d00092cc56a1cf203cc9dde00c5de985bb980aa Mon Sep 17 00:00:00 2001 From: tanghc Date: Mon, 1 Mar 2021 11:12:33 +0800 Subject: [PATCH 2/9] 4.2.7 --- doc/docs/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/docs/README.md b/doc/docs/README.md index af857fe0..aa2509da 100644 --- a/doc/docs/README.md +++ b/doc/docs/README.md @@ -2,6 +2,6 @@ Git地址：[SOP](https://gitee.com/durcframework/SOP) -Q群：167643071 +- QQ群 -![SOP2群](https://images.gitee.com/uploads/images/2020/0203/103119_bdf84eb6_332975.png "SOP2群.png") +1群：167643071（满）、2群：167643071（满）、3群：565894770 \ No newline at end of file From 91f8c82f0e0e66542e5de3d22ab83e99c0e79f3b Mon Sep 17 00:00:00 2001 From: tanghc Date: Tue, 2 Mar 2021 14:28:22 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=80=A7=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 5 +++++ .../storyweb/controller/Example1001_BaseController.java | 2 +- .../gitee/sop/gateway/interceptor/MyRouteInterceptor.java | 8 +++++++- .../src/test/java/com/gitee/sop/test/AllInOneTest.java | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 25a95e36..9242a9ce 100644 --- a/pom.xml +++ b/pom.xml @@ -73,6 +73,11 @@ + + io.projectreactor.netty + reactor-netty + 0.8.19.RELEASE + org.springframework.boot diff --git a/sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java b/sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java index 38737f8a..879d2762 100644 --- a/sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java +++ b/sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java @@ -152,7 +152,7 @@ public class Example1001_BaseController { // 忽略验证 @ApiOperation(value = "忽略签名验证", notes = "忽略签名验证") @Open(value = "story.get.ignore", ignoreValidate = true) - @PostMapping(value = "/get/ignore/v1") + @RequestMapping(value = "/get/ignore/v1") public StoryResult getStory21(@RequestBody StoryParam story) { StoryResult result = new StoryResult(); result.setId((long) story.getId()); diff --git a/sop-gateway/src/main/java/com/gitee/sop/gateway/interceptor/MyRouteInterceptor.java b/sop-gateway/src/main/java/com/gitee/sop/gateway/interceptor/MyRouteInterceptor.java index 4ff60784..8e534a70 100644 --- a/sop-gateway/src/main/java/com/gitee/sop/gateway/interceptor/MyRouteInterceptor.java +++ b/sop-gateway/src/main/java/com/gitee/sop/gateway/interceptor/MyRouteInterceptor.java @@ -5,6 +5,9 @@ import com.gitee.sop.gatewaycommon.interceptor.RouteInterceptorContext; import com.gitee.sop.gatewaycommon.param.ApiParam; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.springframework.web.server.ServerWebExchange; + +import java.net.URI; /** * 演示拦截器 @@ -18,7 +21,10 @@ public class MyRouteInterceptor implements RouteInterceptor { @Override public void preRoute(RouteInterceptorContext context) { ApiParam apiParam = context.getApiParam(); - log.info("请求接口:{}, request_id:{}, app_id:{}, ip:{}", + ServerWebExchange exchange = (ServerWebExchange) context.getRequestContext(); + URI uri = exchange.getRequest().getURI(); + log.info("请求URL:{}, 请求接口:{}, request_id:{}, app_id:{}, ip:{}", + uri, apiParam.fetchNameVersion(), apiParam.fetchRequestId(), apiParam.fetchAppKey(), diff --git a/sop-test/src/test/java/com/gitee/sop/test/AllInOneTest.java b/sop-test/src/test/java/com/gitee/sop/test/AllInOneTest.java index c6e01035..7ee918a8 100644 --- a/sop-test/src/test/java/com/gitee/sop/test/AllInOneTest.java +++ b/sop-test/src/test/java/com/gitee/sop/test/AllInOneTest.java @@ -102,7 +102,7 @@ public class AllInOneTest extends TestBase { .version("1.0") .ignoreSign(true) .bizContent(new BizContent().add("id", "222").add("name", "忽略222")) - .httpMethod(HttpTool.HTTPMethod.POST); + .httpMethod(HttpTool.HTTPMethod.GET); client.execute(requestBuilder); } From 181b881bc536ff187c201299ce7708f9e7f41d33 Mon Sep 17 00:00:00 2001 From: tanghc Date: Tue, 2 Mar 2021 14:34:08 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=80=A7=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java index 76c68aec..16af7f8d 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java @@ -93,7 +93,7 @@ public abstract class BaseExecutorAdapter implements ResultExecutor serviceResult = formatResult(serviceResult); boolean isMergeResult = this.isMergeResult(request); int responseStatus = this.getResponseStatus(request); - this.doAfterRoute(serviceResult, responseStatus, request); +// this.doAfterRoute(serviceResult, responseStatus, request); String finalResult; if (isMergeResult) { Map responseData = this.parseServiceResult(serviceResult, responseStatus, request); From 995d5ba623f3ca06de12f56f7cd088381d88bb6a Mon Sep 17 00:00:00 2001 From: tanghc Date: Tue, 2 Mar 2021 17:41:41 +0800 Subject: [PATCH 5/9] 4.3.0 --- changelog.md | 5 + doc/docs/files/90100_常见问题.md | 8 ++ pom.xml | 20 +--- sop-common/sop-bridge-eureka/pom.xml | 5 +- sop-common/sop-bridge-nacos/pom.xml | 5 +- sop-common/sop-gateway-common/pom.xml | 4 +- .../SopGatewayEnvironmentPostProcessor.java | 9 ++ .../gateway/ServerWebExchangeUtil.java | 10 +- .../controller/ConfigChannelController.java | 7 +- .../GatewayModifyResponseGatewayFilter.java | 95 ++++++++++++++++--- .../gateway/filter/IndexFilter.java | 6 +- .../result/BaseExecutorAdapter.java | 2 +- sop-common/sop-service-common/pom.xml | 4 +- .../configuration/ServiceConfiguration.java | 12 ++- .../resources/META-INF/gateway.properties | 2 + .../main/resources/application-dev.properties | 4 +- 16 files changed, 146 insertions(+), 52 deletions(-) diff --git a/changelog.md b/changelog.md index cc58518b..b6ae81cf 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,10 @@ # changelog +## 4.3.0 + +- 升级`spring-boot/spring-cloud/spring-cloud-alibaba`版本 +- 修复`DataBufferLimitException: Exceeded limit on max bytes to buffer : 262144`问题 + ## 4.2.7 - 修复两个微服务相同path问题 diff --git a/doc/docs/files/90100_常见问题.md b/doc/docs/files/90100_常见问题.md index f7879a1a..67e11668 100644 --- a/doc/docs/files/90100_常见问题.md +++ b/doc/docs/files/90100_常见问题.md @@ -1,5 +1,13 @@ # 常见问题 +## spring cloud gateway [DataBufferLimitException: Exceeded limit on max bytes to buffer : 262144] + +如果POST请求body内容太大可能会报这个错误 + +- 解决版办法 + +网关指定配置`spring.codec.max-in-memory-size=xx`，xx默认是262144，即256K（262144=256*1024） + ## primordials is not defined node版本太高导致与gulp版本不兼容，解决方法：node退回11版本。参考https://blog.csdn.net/zxxzxx23/article/details/103000393 diff --git a/pom.xml b/pom.xml index 9242a9ce..342a50ce 100644 --- a/pom.xml +++ b/pom.xml @@ -2,13 +2,6 @@ - - org.springframework.boot - spring-boot-starter-parent - 2.1.8.RELEASE - - - 4.0.0 com.gitee.sop sop-parent @@ -30,7 +23,7 @@ - 4.2.7-SNAPSHOT + 4.3.0-SNAPSHOT 1.8 UTF-8 @@ -38,12 +31,12 @@ 1.8 - 2.1.8.RELEASE + 2.3.2.RELEASE - Greenwich.SR6 + Hoxton.SR8 - 2.1.2.RELEASE + 2.2.5.RELEASE @@ -73,11 +66,6 @@ - - io.projectreactor.netty - reactor-netty - 0.8.19.RELEASE - org.springframework.boot diff --git a/sop-common/sop-bridge-eureka/pom.xml b/sop-common/sop-bridge-eureka/pom.xml index 65881046..f619aa8b 100644 --- a/sop-common/sop-bridge-eureka/pom.xml +++ b/sop-common/sop-bridge-eureka/pom.xml @@ -4,11 +4,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> com.gitee.sop - sop-parent + sop-common ${revision} - ../../pom.xml + ../pom.xml - 4.0.0 sop-bridge-eureka diff --git a/sop-common/sop-bridge-nacos/pom.xml b/sop-common/sop-bridge-nacos/pom.xml index e30c9732..0b27ec6e 100644 --- a/sop-common/sop-bridge-nacos/pom.xml +++ b/sop-common/sop-bridge-nacos/pom.xml @@ -4,11 +4,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> com.gitee.sop - sop-parent + sop-common ${revision} - ../../pom.xml + ../pom.xml - 4.0.0 sop-bridge-nacos diff --git a/sop-common/sop-gateway-common/pom.xml b/sop-common/sop-gateway-common/pom.xml index 0fd8678f..8289bcf1 100644 --- a/sop-common/sop-gateway-common/pom.xml +++ b/sop-common/sop-gateway-common/pom.xml @@ -4,9 +4,9 @@ com.gitee.sop - sop-parent + sop-common ${revision} - ../../pom.xml + ../pom.xml 4.0.0 diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java index 6f12412c..476f3083 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java @@ -7,6 +7,7 @@ import org.springframework.core.env.PropertiesPropertySource; import org.springframework.core.env.PropertySource; import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.Resource; +import org.springframework.util.unit.DataSize; import java.io.IOException; import java.util.Properties; @@ -31,9 +32,17 @@ public class SopGatewayEnvironmentPostProcessor implements EnvironmentPostProces } try { properties.load(resource.getInputStream()); + this.initMemSize(properties); return new PropertiesPropertySource(resource.getFilename(), properties); } catch (IOException ex) { throw new IllegalStateException("加载配置文件失败" + resource, ex); } } + + private void initMemSize(Properties properties) { + String size = properties.getProperty("spring.servlet.multipart.max-file-size", "10M"); + DataSize dataSize = DataSize.parse(size); + properties.putIfAbsent("spring.codec.max-in-memory-size", dataSize.toBytes()); + } + } diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/ServerWebExchangeUtil.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/ServerWebExchangeUtil.java index 49de630a..303650ae 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/ServerWebExchangeUtil.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/ServerWebExchangeUtil.java @@ -16,13 +16,12 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.MediaType; -import org.springframework.http.codec.HttpMessageReader; +import org.springframework.http.codec.ServerCodecConfigurer; import org.springframework.http.converter.FormHttpMessageConverter; import org.springframework.http.server.reactive.ServerHttpRequest; import org.springframework.util.MultiValueMap; import org.springframework.web.multipart.MultipartHttpServletRequest; import org.springframework.web.multipart.commons.CommonsMultipartResolver; -import org.springframework.web.reactive.function.server.HandlerStrategies; import org.springframework.web.reactive.function.server.ServerRequest; import org.springframework.web.server.ServerWebExchange; import org.springframework.web.server.WebFilterChain; @@ -47,8 +46,6 @@ public class ServerWebExchangeUtil { private static final FormHttpMessageConverter formHttpMessageConverter = new FormHttpMessageConverter(); - private static final List> messageReaders = HandlerStrategies.withDefaults().messageReaders(); - /** * 重定向 * @@ -70,10 +67,11 @@ public class ServerWebExchangeUtil { * 构建一个接受请求体的request * * @param exchange exchange + * @param codecConfigurer codecConfigurer * @return 返回ServerRequest */ - public static ServerRequest createReadBodyRequest(ServerWebExchange exchange) { - return ServerRequest.create(exchange, messageReaders); + public static ServerRequest createReadBodyRequest(ServerWebExchange exchange, ServerCodecConfigurer codecConfigurer) { + return ServerRequest.create(exchange, codecConfigurer.getReaders()); } public static String getRestfulPath(String path, String prefix) { diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/controller/ConfigChannelController.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/controller/ConfigChannelController.java index 1ccc64e1..47877065 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/controller/ConfigChannelController.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/controller/ConfigChannelController.java @@ -14,7 +14,9 @@ import com.gitee.sop.gatewaycommon.manager.RouteConfigManager; import com.gitee.sop.gatewaycommon.secret.IsvManager; import com.gitee.sop.gatewaycommon.util.RequestUtil; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.codec.ServerCodecConfigurer; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.reactive.function.server.ServerRequest; @@ -45,9 +47,12 @@ public class ConfigChannelController { @Value("${sop.secret}") private String secret; + @Autowired + private ServerCodecConfigurer codecConfigurer; + @PostMapping("/sop/configChannelMsg") public Mono configChannel(ServerWebExchange exchange) { - ServerRequest serverRequest = ServerWebExchangeUtil.createReadBodyRequest(exchange); + ServerRequest serverRequest = ServerWebExchangeUtil.createReadBodyRequest(exchange, codecConfigurer); // 读取请求体中的内容 return serverRequest.bodyToMono(String.class) .flatMap(requestJson -> { diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/GatewayModifyResponseGatewayFilter.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/GatewayModifyResponseGatewayFilter.java index 4d3bc4c1..ca45c1ac 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/GatewayModifyResponseGatewayFilter.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/GatewayModifyResponseGatewayFilter.java @@ -5,36 +5,58 @@ import com.gitee.sop.gatewaycommon.bean.SopConstants; import com.gitee.sop.gatewaycommon.result.ResultExecutor; import org.apache.commons.lang3.StringUtils; import org.reactivestreams.Publisher; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cloud.gateway.filter.GatewayFilterChain; import org.springframework.cloud.gateway.filter.GlobalFilter; import org.springframework.cloud.gateway.filter.NettyWriteResponseFilter; import org.springframework.cloud.gateway.filter.factory.rewrite.CachedBodyOutputMessage; +import org.springframework.cloud.gateway.filter.factory.rewrite.MessageBodyEncoder; import org.springframework.cloud.gateway.support.BodyInserterContext; -import org.springframework.cloud.gateway.support.DefaultClientResponse; import org.springframework.core.Ordered; import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.core.io.buffer.DataBufferFactory; +import org.springframework.core.io.buffer.DataBufferUtils; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseCookie; import org.springframework.http.client.reactive.ClientHttpResponse; +import org.springframework.http.codec.ServerCodecConfigurer; +import org.springframework.http.server.reactive.ServerHttpResponse; import org.springframework.http.server.reactive.ServerHttpResponseDecorator; import org.springframework.util.MultiValueMap; import org.springframework.web.reactive.function.BodyInserter; import org.springframework.web.reactive.function.BodyInserters; -import org.springframework.web.reactive.function.client.ExchangeStrategies; +import org.springframework.web.reactive.function.client.ClientResponse; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; +import javax.annotation.PostConstruct; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.function.Function.identity; import static org.springframework.cloud.gateway.support.ServerWebExchangeUtils.ORIGINAL_RESPONSE_CONTENT_TYPE_ATTR; /** * 修改返回结果 + * * @author tanghc */ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered { + @Autowired + private ServerCodecConfigurer codecConfigurer; + @Autowired + private Set bodyEncoders; + + private Map messageBodyEncoders; + @Override @SuppressWarnings("unchecked") public Mono filter(ServerWebExchange exchange, GatewayFilterChain chain) { @@ -54,12 +76,14 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered } Class inClass = String.class; Class outClass = String.class; + HttpHeaders httpHeaders = new HttpHeaders(); - //explicitly add it in this way instead of 'httpHeaders.setContentType(originalResponseContentType)' - //this will prevent exception in case of using non-standard media types like "Content-Type: image" + // explicitly add it in this way instead of + // 'httpHeaders.setContentType(originalResponseContentType)' + // this will prevent exception in case of using non-standard media + // types like "Content-Type: image" httpHeaders.add(HttpHeaders.CONTENT_TYPE, originalResponseContentType); - ResponseAdapter responseAdapter = new ResponseAdapter(body, httpHeaders); - DefaultClientResponse clientResponse = new DefaultClientResponse(responseAdapter, ExchangeStrategies.withDefaults()); + ClientResponse clientResponse = prepareClientResponse(exchange, body, httpHeaders); //TODO: flux or mono Mono modifiedBody = clientResponse.bodyToMono(inClass) @@ -72,16 +96,21 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered return Mono.just(ret); }); - BodyInserter bodyInserter = BodyInserters.fromPublisher(modifiedBody, outClass); - CachedBodyOutputMessage outputMessage = new CachedBodyOutputMessage(exchange, exchange.getResponse().getHeaders()); + BodyInserter bodyInserter = BodyInserters.fromPublisher(modifiedBody, + outClass); + CachedBodyOutputMessage outputMessage = new CachedBodyOutputMessage(exchange, + exchange.getResponse().getHeaders()); return bodyInserter.insert(outputMessage, new BodyInserterContext()) .then(Mono.defer(() -> { - Flux messageBody = outputMessage.getBody(); + Mono messageBody = writeBody(getDelegate(), + outputMessage, outClass); HttpHeaders headers = getDelegate().getHeaders(); - if (!headers.containsKey(HttpHeaders.TRANSFER_ENCODING)) { - messageBody = messageBody.doOnNext(data -> headers.setContentLength(data.readableByteCount())); + if (!headers.containsKey(HttpHeaders.TRANSFER_ENCODING) + || headers.containsKey(HttpHeaders.CONTENT_LENGTH)) { + messageBody = messageBody.doOnNext(data -> headers + .setContentLength(data.readableByteCount())); } - //TODO: use isStreamingMediaType? + // TODO: fail if isStreamingMediaType? return getDelegate().writeWith(messageBody); })); } @@ -96,6 +125,42 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered return chain.filter(exchange.mutate().response(responseDecorator).build()); } + private ClientResponse prepareClientResponse( + ServerWebExchange exchange, + Publisher body, + HttpHeaders httpHeaders + ) { + ClientResponse.Builder builder; + builder = ClientResponse.create(exchange.getResponse().getStatusCode(), codecConfigurer.getReaders()); + return builder.headers(headers -> headers.putAll(httpHeaders)) + .body(Flux.from(body)).build(); + } + + private Mono writeBody(ServerHttpResponse httpResponse, + CachedBodyOutputMessage message, Class outClass) { + Mono response = DataBufferUtils.join(message.getBody()); + if (byte[].class.isAssignableFrom(outClass)) { + return response; + } + + List encodingHeaders = httpResponse.getHeaders() + .getOrEmpty(HttpHeaders.CONTENT_ENCODING); + for (String encoding : encodingHeaders) { + MessageBodyEncoder encoder = messageBodyEncoders.get(encoding); + if (encoder != null) { + DataBufferFactory dataBufferFactory = httpResponse.bufferFactory(); + response = response.publishOn(Schedulers.parallel()).map(buffer -> { + byte[] encodedResponse = encoder.encode(buffer); + DataBufferUtils.release(buffer); + return encodedResponse; + }).map(dataBufferFactory::wrap); + break; + } + } + + return response; + } + @Override public int getOrder() { return NettyWriteResponseFilter.WRITE_RESPONSE_FILTER_ORDER - 1; @@ -140,4 +205,10 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered return null; } } + + @PostConstruct + public void after() { + this.messageBodyEncoders = bodyEncoders == null ? Collections.emptyMap() : bodyEncoders.stream() + .collect(Collectors.toMap(MessageBodyEncoder::encodingType, identity())); + } } \ No newline at end of file diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/IndexFilter.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/IndexFilter.java index 0e15973a..0c261675 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/IndexFilter.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/IndexFilter.java @@ -20,6 +20,7 @@ import org.springframework.core.annotation.Order; import org.springframework.core.io.buffer.DataBuffer; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; +import org.springframework.http.codec.ServerCodecConfigurer; import org.springframework.http.server.reactive.ServerHttpRequest; import org.springframework.http.server.reactive.ServerHttpRequestDecorator; import org.springframework.web.reactive.function.BodyInserter; @@ -45,6 +46,9 @@ import java.util.Objects; @Order(Ordered.HIGHEST_PRECEDENCE) public class IndexFilter implements WebFilter { + @Autowired + private ServerCodecConfigurer codecConfigurer; + /** 路径白名单 */ private static final List PATH_WHITE_LIST = Arrays.asList( "/sop", "/actuator" @@ -78,7 +82,7 @@ public class IndexFilter implements WebFilter { } if (Objects.equals(path, indexPath) || "".equals(path)) { if (request.getMethod() == HttpMethod.POST) { - ServerRequest serverRequest = ServerWebExchangeUtil.createReadBodyRequest(exchange); + ServerRequest serverRequest = ServerWebExchangeUtil.createReadBodyRequest(exchange, codecConfigurer); // 读取请求体中的内容 Mono modifiedBody = serverRequest.bodyToMono(byte[].class) .switchIfEmpty(Mono.just("".getBytes())) diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java index 16af7f8d..76c68aec 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java @@ -93,7 +93,7 @@ public abstract class BaseExecutorAdapter implements ResultExecutor serviceResult = formatResult(serviceResult); boolean isMergeResult = this.isMergeResult(request); int responseStatus = this.getResponseStatus(request); -// this.doAfterRoute(serviceResult, responseStatus, request); + this.doAfterRoute(serviceResult, responseStatus, request); String finalResult; if (isMergeResult) { Map responseData = this.parseServiceResult(serviceResult, responseStatus, request); diff --git a/sop-common/sop-service-common/pom.xml b/sop-common/sop-service-common/pom.xml index 071288f5..6c1b3a37 100644 --- a/sop-common/sop-service-common/pom.xml +++ b/sop-common/sop-service-common/pom.xml @@ -5,9 +5,9 @@ com.gitee.sop - sop-parent + sop-common ${revision} - ../../pom.xml + ../pom.xml 4.0.0 diff --git a/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/configuration/ServiceConfiguration.java b/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/configuration/ServiceConfiguration.java index 1f302e88..460497da 100644 --- a/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/configuration/ServiceConfiguration.java +++ b/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/configuration/ServiceConfiguration.java @@ -1,6 +1,7 @@ package com.gitee.sop.servercommon.configuration; import com.alibaba.cloud.nacos.NacosDiscoveryProperties; +import com.alibaba.cloud.nacos.NacosServiceManager; import com.alibaba.cloud.nacos.discovery.NacosWatch; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.ObjectProvider; @@ -8,7 +9,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.core.env.Environment; -import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import java.util.Map; @@ -21,7 +22,12 @@ public class ServiceConfiguration extends SpringmvcConfiguration { @Bean @ConditionalOnMissingBean @ConditionalOnProperty("spring.cloud.nacos.discovery.server-addr") - public NacosWatch nacosWatch(NacosDiscoveryProperties nacosDiscoveryProperties, ObjectProvider taskScheduler, Environment environment) { + public NacosWatch nacosWatch( + NacosServiceManager nacosServiceManager, + NacosDiscoveryProperties nacosDiscoveryProperties, + ObjectProvider taskScheduler, + Environment environment + ) { Map metadata = nacosDiscoveryProperties.getMetadata(); String contextPath = environment.getProperty(METADATA_SERVER_CONTEXT_PATH); // 将context-path信息加入到metadata中 @@ -31,7 +37,7 @@ public class ServiceConfiguration extends SpringmvcConfiguration { // 在元数据中新增启动时间，不能修改这个值，不然网关拉取接口会有问题 // 如果没有这个值，网关会忽略这个服务 metadata.put("server.startup-time", String.valueOf(System.currentTimeMillis())); - return new NacosWatch(nacosDiscoveryProperties, taskScheduler); + return new NacosWatch(nacosServiceManager, nacosDiscoveryProperties, taskScheduler); } } diff --git a/sop-gateway/src/main/resources/META-INF/gateway.properties b/sop-gateway/src/main/resources/META-INF/gateway.properties index 652167d2..399730eb 100644 --- a/sop-gateway/src/main/resources/META-INF/gateway.properties +++ b/sop-gateway/src/main/resources/META-INF/gateway.properties @@ -34,6 +34,8 @@ spring.datasource.hikari.max-lifetime=500000 spring.cloud.gateway.discovery.locator.lower-case-service-id=true spring.cloud.gateway.discovery.locator.enabled=true +# \u54CD\u5E94\u8D85\u65F6\uFF0C\u9ED8\u8BA410\u79D2\uFF0810000\uFF09 +spring.cloud.gateway.httpclient.response-timeout=10000 # \u4E0D\u7528\u6539 mybatis.mapper-locations=classpath:mybatis/mapper/*.xml diff --git a/sop-gateway/src/main/resources/application-dev.properties b/sop-gateway/src/main/resources/application-dev.properties index 4a154c25..8afebce4 100644 --- a/sop-gateway/src/main/resources/application-dev.properties +++ b/sop-gateway/src/main/resources/application-dev.properties @@ -12,7 +12,7 @@ register.url=127.0.0.1:8848 # \u4E0A\u4F20\u6587\u4EF6\u6700\u5927\u5BB9\u91CF\uFF0C\u9ED8\u8BA410MB spring.servlet.multipart.max-file-size=10MB -# \u54CD\u5E94\u8D85\u65F6\uFF0C\u9ED8\u8BA45\u79D2\uFF085000\uFF09 -#spring.cloud.gateway.httpclient.response-timeout=5000 +# \u54CD\u5E94\u8D85\u65F6\uFF0C\u9ED8\u8BA410\u79D2\uFF0810000\uFF09 +#spring.cloud.gateway.httpclient.response-timeout=10000 logging.level.com.gitee=debug From d0fe92a96cbbbd7bffdc4b41c3a3a50e9a4610f8 Mon Sep 17 00:00:00 2001 From: tanghc Date: Tue, 2 Mar 2021 17:59:14 +0800 Subject: [PATCH 6/9] 4.3.0 --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 342a50ce..8f820976 100644 --- a/pom.xml +++ b/pom.xml @@ -2,6 +2,12 @@ + + org.springframework.boot + spring-boot-starter-parent + 2.3.2.RELEASE + + 4.0.0 com.gitee.sop sop-parent From 888825ba50f24fad5c5dbc609e3d6329a19cfe5b Mon Sep 17 00:00:00 2001 From: tanghc Date: Tue, 2 Mar 2021 19:03:48 +0800 Subject: [PATCH 7/9] 4.3.0 --- changelog.md | 1 + doc/docs/files/90100_常见问题.md | 8 ++++++++ .../src/main/resources/META-INF/gateway.properties | 1 + 3 files changed, 10 insertions(+) diff --git a/changelog.md b/changelog.md index b6ae81cf..6f0e378a 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,7 @@ - 升级`spring-boot/spring-cloud/spring-cloud-alibaba`版本 - 修复`DataBufferLimitException: Exceeded limit on max bytes to buffer : 262144`问题 +- 修复压测出现`ClosedChannelException`问题 ## 4.2.7 diff --git a/doc/docs/files/90100_常见问题.md b/doc/docs/files/90100_常见问题.md index 67e11668..917a2974 100644 --- a/doc/docs/files/90100_常见问题.md +++ b/doc/docs/files/90100_常见问题.md @@ -1,5 +1,13 @@ # 常见问题 +## Connection has been closed BEFORE send operation + +压测时出现`Connection has been closed BEFORE send operation`，`java.nio.channels.ClosedChannelException`之类的错误 + +- 解决办法 + +尝试调大内存分配1G或2G，`-Xms1g -Xmx1g` + ## spring cloud gateway [DataBufferLimitException: Exceeded limit on max bytes to buffer : 262144] 如果POST请求body内容太大可能会报这个错误 diff --git a/sop-gateway/src/main/resources/META-INF/gateway.properties b/sop-gateway/src/main/resources/META-INF/gateway.properties index 399730eb..04a47379 100644 --- a/sop-gateway/src/main/resources/META-INF/gateway.properties +++ b/sop-gateway/src/main/resources/META-INF/gateway.properties @@ -36,6 +36,7 @@ spring.cloud.gateway.discovery.locator.lower-case-service-id=true spring.cloud.gateway.discovery.locator.enabled=true # \u54CD\u5E94\u8D85\u65F6\uFF0C\u9ED8\u8BA410\u79D2\uFF0810000\uFF09 spring.cloud.gateway.httpclient.response-timeout=10000 +spring.cloud.gateway.httpclient.pool.max-idle-time=60000 # \u4E0D\u7528\u6539 mybatis.mapper-locations=classpath:mybatis/mapper/*.xml From 73fbd7a072f3c72bf023e2dbbada90e1d41680c3 Mon Sep 17 00:00:00 2001 From: tanghc Date: Wed, 3 Mar 2021 09:33:05 +0800 Subject: [PATCH 8/9] 4.3.0 --- changelog.md | 2 +- .../config/SopGatewayEnvironmentPostProcessor.java | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index 6f0e378a..69a2dd87 100644 --- a/changelog.md +++ b/changelog.md @@ -4,7 +4,7 @@ - 升级`spring-boot/spring-cloud/spring-cloud-alibaba`版本 - 修复`DataBufferLimitException: Exceeded limit on max bytes to buffer : 262144`问题 -- 修复压测出现`ClosedChannelException`问题 +- 修复压测出现`ClosedChannelException`，`Connection has been closed BEFORE response`问题 ## 4.2.7 diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java index 476f3083..d92ec8e1 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java @@ -40,9 +40,13 @@ public class SopGatewayEnvironmentPostProcessor implements EnvironmentPostProces } private void initMemSize(Properties properties) { - String size = properties.getProperty("spring.servlet.multipart.max-file-size", "10M"); - DataSize dataSize = DataSize.parse(size); - properties.putIfAbsent("spring.codec.max-in-memory-size", dataSize.toBytes()); + String maxFileSizeKey = "spring.servlet.multipart.max-file-size"; + String maxMemSizeKey = "spring.codec.max-in-memory-size"; + String defaultSize = "10M"; + String maxUploadSize = properties.getProperty(maxFileSizeKey, defaultSize); + String maxBodySize = properties.getProperty(maxMemSizeKey, defaultSize); + long size = Math.max(DataSize.parse(maxUploadSize).toBytes(), DataSize.parse(maxBodySize).toBytes()); + properties.put(maxMemSizeKey, size); } } From 008c0f830f73dd8d9abbc0c5869be7d3d75bb8bb Mon Sep 17 00:00:00 2001 From: tanghc Date: Wed, 3 Mar 2021 09:51:57 +0800 Subject: [PATCH 9/9] 4.3.0 --- .../config/SopGatewayEnvironmentPostProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java index d92ec8e1..233d843f 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java @@ -42,7 +42,7 @@ public class SopGatewayEnvironmentPostProcessor implements EnvironmentPostProces private void initMemSize(Properties properties) { String maxFileSizeKey = "spring.servlet.multipart.max-file-size"; String maxMemSizeKey = "spring.codec.max-in-memory-size"; - String defaultSize = "10M"; + String defaultSize = "10MB"; String maxUploadSize = properties.getProperty(maxFileSizeKey, defaultSize); String maxBodySize = properties.getProperty(maxMemSizeKey, defaultSize); long size = Math.max(DataSize.parse(maxUploadSize).toBytes(), DataSize.parse(maxBodySize).toBytes());