Merge branch 'master' into eureka

# Conflicts:
#	sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/configuration/ServiceConfiguration.java
eureka
tanghc 4 years ago
commit 7aae4b08ff
  1. 4
      README.md
  2. 6
      changelog.md
  3. 4
      doc/docs/README.md
  4. 16
      doc/docs/files/90100_常见问题.md
  5. 11
      pom.xml
  6. 5
      sop-common/sop-bridge-eureka/pom.xml
  7. 5
      sop-common/sop-bridge-nacos/pom.xml
  8. 4
      sop-common/sop-gateway-common/pom.xml
  9. 13
      sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/SopGatewayEnvironmentPostProcessor.java
  10. 10
      sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/ServerWebExchangeUtil.java
  11. 7
      sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/controller/ConfigChannelController.java
  12. 95
      sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/GatewayModifyResponseGatewayFilter.java
  13. 6
      sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/filter/IndexFilter.java
  14. 4
      sop-common/sop-service-common/pom.xml
  15. 2
      sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java
  16. 8
      sop-gateway/src/main/java/com/gitee/sop/gateway/interceptor/MyRouteInterceptor.java
  17. 3
      sop-gateway/src/main/resources/META-INF/gateway.properties
  18. 4
      sop-gateway/src/main/resources/application-dev.properties
  19. 2
      sop-test/src/test/java/com/gitee/sop/test/AllInOneTest.java

@ -143,6 +143,6 @@ System.out.println(responseData);
## 沟通交流
Q群:167643071
- QQ群
![SOP2群](https://images.gitee.com/uploads/images/2020/0203/103119_bdf84eb6_332975.png "SOP2群.png")
1群:167643071(满)、2群:167643071(满)、3群:565894770

@ -1,5 +1,11 @@
# changelog
## 4.3.0
- 升级`spring-boot/spring-cloud/spring-cloud-alibaba`版本
- 修复`DataBufferLimitException: Exceeded limit on max bytes to buffer : 262144`问题
- 修复压测出现`ClosedChannelException`,`Connection has been closed BEFORE response`问题
## 4.2.7
- 修复两个微服务相同path问题

@ -2,6 +2,6 @@
Git地址:[SOP](https://gitee.com/durcframework/SOP)
Q群:167643071
- QQ群
![SOP2群](https://images.gitee.com/uploads/images/2020/0203/103119_bdf84eb6_332975.png "SOP2群.png")
1群:167643071(满)、2群:167643071(满)、3群:565894770

@ -1,5 +1,21 @@
# 常见问题
## Connection has been closed BEFORE send operation
压测时出现`Connection has been closed BEFORE send operation`,`java.nio.channels.ClosedChannelException`之类的错误
- 解决办法
尝试调大内存分配1G或2G,`-Xms1g -Xmx1g`
## spring cloud gateway [DataBufferLimitException: Exceeded limit on max bytes to buffer : 262144]
如果POST请求body内容太大可能会报这个错误
- 解决版办法
网关指定配置`spring.codec.max-in-memory-size=xx`,xx默认是262144,即256K(262144=256*1024)
## primordials is not defined
node版本太高导致与gulp版本不兼容,解决方法:node退回11版本。参考https://blog.csdn.net/zxxzxx23/article/details/103000393

@ -5,10 +5,9 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.8.RELEASE</version>
<version>2.3.2.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.gitee.sop</groupId>
<artifactId>sop-parent</artifactId>
@ -30,7 +29,7 @@
<properties>
<!-- Project revision -->
<revision>4.2.7-SNAPSHOT</revision>
<revision>4.3.0-SNAPSHOT</revision>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@ -38,12 +37,12 @@
<maven.compiler.target>1.8</maven.compiler.target>
<!-- springboot 版本-->
<spring-boot.version>2.1.8.RELEASE</spring-boot.version>
<spring-boot.version>2.3.2.RELEASE</spring-boot.version>
<!-- spring cloud 版本 -->
<spring-cloud.version>Greenwich.SR6</spring-cloud.version>
<spring-cloud.version>Hoxton.SR8</spring-cloud.version>
<!-- spring cloud alibaba 版本 -->
<!-- 具体版本对应关系见:https://github.com/alibaba/spring-cloud-alibaba/wiki/%E7%89%88%E6%9C%AC%E8%AF%B4%E6%98%8E -->
<spring-cloud-alibaba.version>2.1.2.RELEASE</spring-cloud-alibaba.version>
<spring-cloud-alibaba.version>2.2.5.RELEASE</spring-cloud-alibaba.version>
<!-- Logging -->

@ -4,11 +4,10 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>com.gitee.sop</groupId>
<artifactId>sop-parent</artifactId>
<artifactId>sop-common</artifactId>
<version>${revision}</version>
<relativePath>../../pom.xml</relativePath> <!-- lookup parent from repository -->
<relativePath>../pom.xml</relativePath> <!-- lookup parent from repository -->
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>sop-bridge-eureka</artifactId>

@ -4,11 +4,10 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>com.gitee.sop</groupId>
<artifactId>sop-parent</artifactId>
<artifactId>sop-common</artifactId>
<version>${revision}</version>
<relativePath>../../pom.xml</relativePath> <!-- lookup parent from repository -->
<relativePath>../pom.xml</relativePath> <!-- lookup parent from repository -->
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>sop-bridge-nacos</artifactId>

@ -4,9 +4,9 @@
<parent>
<groupId>com.gitee.sop</groupId>
<artifactId>sop-parent</artifactId>
<artifactId>sop-common</artifactId>
<version>${revision}</version>
<relativePath>../../pom.xml</relativePath> <!-- lookup parent from repository -->
<relativePath>../pom.xml</relativePath> <!-- lookup parent from repository -->
</parent>
<modelVersion>4.0.0</modelVersion>

@ -7,6 +7,7 @@ import org.springframework.core.env.PropertiesPropertySource;
import org.springframework.core.env.PropertySource;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.springframework.util.unit.DataSize;
import java.io.IOException;
import java.util.Properties;
@ -31,9 +32,21 @@ public class SopGatewayEnvironmentPostProcessor implements EnvironmentPostProces
}
try {
properties.load(resource.getInputStream());
this.initMemSize(properties);
return new PropertiesPropertySource(resource.getFilename(), properties);
} catch (IOException ex) {
throw new IllegalStateException("加载配置文件失败" + resource, ex);
}
}
private void initMemSize(Properties properties) {
String maxFileSizeKey = "spring.servlet.multipart.max-file-size";
String maxMemSizeKey = "spring.codec.max-in-memory-size";
String defaultSize = "10MB";
String maxUploadSize = properties.getProperty(maxFileSizeKey, defaultSize);
String maxBodySize = properties.getProperty(maxMemSizeKey, defaultSize);
long size = Math.max(DataSize.parse(maxUploadSize).toBytes(), DataSize.parse(maxBodySize).toBytes());
properties.put(maxMemSizeKey, size);
}
}

@ -16,13 +16,12 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.codec.HttpMessageReader;
import org.springframework.http.codec.ServerCodecConfigurer;
import org.springframework.http.converter.FormHttpMessageConverter;
import org.springframework.http.server.reactive.ServerHttpRequest;
import org.springframework.util.MultiValueMap;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import org.springframework.web.multipart.commons.CommonsMultipartResolver;
import org.springframework.web.reactive.function.server.HandlerStrategies;
import org.springframework.web.reactive.function.server.ServerRequest;
import org.springframework.web.server.ServerWebExchange;
import org.springframework.web.server.WebFilterChain;
@ -47,8 +46,6 @@ public class ServerWebExchangeUtil {
private static final FormHttpMessageConverter formHttpMessageConverter = new FormHttpMessageConverter();
private static final List<HttpMessageReader<?>> messageReaders = HandlerStrategies.withDefaults().messageReaders();
/**
* 重定向
*
@ -70,10 +67,11 @@ public class ServerWebExchangeUtil {
* 构建一个接受请求体的request
*
* @param exchange exchange
* @param codecConfigurer codecConfigurer
* @return 返回ServerRequest
*/
public static ServerRequest createReadBodyRequest(ServerWebExchange exchange) {
return ServerRequest.create(exchange, messageReaders);
public static ServerRequest createReadBodyRequest(ServerWebExchange exchange, ServerCodecConfigurer codecConfigurer) {
return ServerRequest.create(exchange, codecConfigurer.getReaders());
}
public static String getRestfulPath(String path, String prefix) {

@ -14,7 +14,9 @@ import com.gitee.sop.gatewaycommon.manager.RouteConfigManager;
import com.gitee.sop.gatewaycommon.secret.IsvManager;
import com.gitee.sop.gatewaycommon.util.RequestUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.codec.ServerCodecConfigurer;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.reactive.function.server.ServerRequest;
@ -45,9 +47,12 @@ public class ConfigChannelController {
@Value("${sop.secret}")
private String secret;
@Autowired
private ServerCodecConfigurer codecConfigurer;
@PostMapping("/sop/configChannelMsg")
public Mono<String> configChannel(ServerWebExchange exchange) {
ServerRequest serverRequest = ServerWebExchangeUtil.createReadBodyRequest(exchange);
ServerRequest serverRequest = ServerWebExchangeUtil.createReadBodyRequest(exchange, codecConfigurer);
// 读取请求体中的内容
return serverRequest.bodyToMono(String.class)
.flatMap(requestJson -> {

@ -5,36 +5,58 @@ import com.gitee.sop.gatewaycommon.bean.SopConstants;
import com.gitee.sop.gatewaycommon.result.ResultExecutor;
import org.apache.commons.lang3.StringUtils;
import org.reactivestreams.Publisher;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cloud.gateway.filter.GatewayFilterChain;
import org.springframework.cloud.gateway.filter.GlobalFilter;
import org.springframework.cloud.gateway.filter.NettyWriteResponseFilter;
import org.springframework.cloud.gateway.filter.factory.rewrite.CachedBodyOutputMessage;
import org.springframework.cloud.gateway.filter.factory.rewrite.MessageBodyEncoder;
import org.springframework.cloud.gateway.support.BodyInserterContext;
import org.springframework.cloud.gateway.support.DefaultClientResponse;
import org.springframework.core.Ordered;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferFactory;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseCookie;
import org.springframework.http.client.reactive.ClientHttpResponse;
import org.springframework.http.codec.ServerCodecConfigurer;
import org.springframework.http.server.reactive.ServerHttpResponse;
import org.springframework.http.server.reactive.ServerHttpResponseDecorator;
import org.springframework.util.MultiValueMap;
import org.springframework.web.reactive.function.BodyInserter;
import org.springframework.web.reactive.function.BodyInserters;
import org.springframework.web.reactive.function.client.ExchangeStrategies;
import org.springframework.web.reactive.function.client.ClientResponse;
import org.springframework.web.server.ServerWebExchange;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import javax.annotation.PostConstruct;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import static java.util.function.Function.identity;
import static org.springframework.cloud.gateway.support.ServerWebExchangeUtils.ORIGINAL_RESPONSE_CONTENT_TYPE_ATTR;
/**
* 修改返回结果
*
* @author tanghc
*/
public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered {
@Autowired
private ServerCodecConfigurer codecConfigurer;
@Autowired
private Set<MessageBodyEncoder> bodyEncoders;
private Map<String, MessageBodyEncoder> messageBodyEncoders;
@Override
@SuppressWarnings("unchecked")
public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
@ -54,12 +76,14 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered
}
Class inClass = String.class;
Class outClass = String.class;
HttpHeaders httpHeaders = new HttpHeaders();
//explicitly add it in this way instead of 'httpHeaders.setContentType(originalResponseContentType)'
//this will prevent exception in case of using non-standard media types like "Content-Type: image"
// explicitly add it in this way instead of
// 'httpHeaders.setContentType(originalResponseContentType)'
// this will prevent exception in case of using non-standard media
// types like "Content-Type: image"
httpHeaders.add(HttpHeaders.CONTENT_TYPE, originalResponseContentType);
ResponseAdapter responseAdapter = new ResponseAdapter(body, httpHeaders);
DefaultClientResponse clientResponse = new DefaultClientResponse(responseAdapter, ExchangeStrategies.withDefaults());
ClientResponse clientResponse = prepareClientResponse(exchange, body, httpHeaders);
//TODO: flux or mono
Mono modifiedBody = clientResponse.bodyToMono(inClass)
@ -72,16 +96,21 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered
return Mono.just(ret);
});
BodyInserter bodyInserter = BodyInserters.fromPublisher(modifiedBody, outClass);
CachedBodyOutputMessage outputMessage = new CachedBodyOutputMessage(exchange, exchange.getResponse().getHeaders());
BodyInserter bodyInserter = BodyInserters.fromPublisher(modifiedBody,
outClass);
CachedBodyOutputMessage outputMessage = new CachedBodyOutputMessage(exchange,
exchange.getResponse().getHeaders());
return bodyInserter.insert(outputMessage, new BodyInserterContext())
.then(Mono.defer(() -> {
Flux<DataBuffer> messageBody = outputMessage.getBody();
Mono<DataBuffer> messageBody = writeBody(getDelegate(),
outputMessage, outClass);
HttpHeaders headers = getDelegate().getHeaders();
if (!headers.containsKey(HttpHeaders.TRANSFER_ENCODING)) {
messageBody = messageBody.doOnNext(data -> headers.setContentLength(data.readableByteCount()));
if (!headers.containsKey(HttpHeaders.TRANSFER_ENCODING)
|| headers.containsKey(HttpHeaders.CONTENT_LENGTH)) {
messageBody = messageBody.doOnNext(data -> headers
.setContentLength(data.readableByteCount()));
}
//TODO: use isStreamingMediaType?
// TODO: fail if isStreamingMediaType?
return getDelegate().writeWith(messageBody);
}));
}
@ -96,6 +125,42 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered
return chain.filter(exchange.mutate().response(responseDecorator).build());
}
private ClientResponse prepareClientResponse(
ServerWebExchange exchange,
Publisher<? extends DataBuffer> body,
HttpHeaders httpHeaders
) {
ClientResponse.Builder builder;
builder = ClientResponse.create(exchange.getResponse().getStatusCode(), codecConfigurer.getReaders());
return builder.headers(headers -> headers.putAll(httpHeaders))
.body(Flux.from(body)).build();
}
private Mono<DataBuffer> writeBody(ServerHttpResponse httpResponse,
CachedBodyOutputMessage message, Class<?> outClass) {
Mono<DataBuffer> response = DataBufferUtils.join(message.getBody());
if (byte[].class.isAssignableFrom(outClass)) {
return response;
}
List<String> encodingHeaders = httpResponse.getHeaders()
.getOrEmpty(HttpHeaders.CONTENT_ENCODING);
for (String encoding : encodingHeaders) {
MessageBodyEncoder encoder = messageBodyEncoders.get(encoding);
if (encoder != null) {
DataBufferFactory dataBufferFactory = httpResponse.bufferFactory();
response = response.publishOn(Schedulers.parallel()).map(buffer -> {
byte[] encodedResponse = encoder.encode(buffer);
DataBufferUtils.release(buffer);
return encodedResponse;
}).map(dataBufferFactory::wrap);
break;
}
}
return response;
}
@Override
public int getOrder() {
return NettyWriteResponseFilter.WRITE_RESPONSE_FILTER_ORDER - 1;
@ -140,4 +205,10 @@ public class GatewayModifyResponseGatewayFilter implements GlobalFilter, Ordered
return null;
}
}
@PostConstruct
public void after() {
this.messageBodyEncoders = bodyEncoders == null ? Collections.emptyMap() : bodyEncoders.stream()
.collect(Collectors.toMap(MessageBodyEncoder::encodingType, identity()));
}
}

@ -20,6 +20,7 @@ import org.springframework.core.annotation.Order;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.codec.ServerCodecConfigurer;
import org.springframework.http.server.reactive.ServerHttpRequest;
import org.springframework.http.server.reactive.ServerHttpRequestDecorator;
import org.springframework.web.reactive.function.BodyInserter;
@ -45,6 +46,9 @@ import java.util.Objects;
@Order(Ordered.HIGHEST_PRECEDENCE)
public class IndexFilter implements WebFilter {
@Autowired
private ServerCodecConfigurer codecConfigurer;
/** 路径白名单 */
private static final List<String> PATH_WHITE_LIST = Arrays.asList(
"/sop", "/actuator"
@ -78,7 +82,7 @@ public class IndexFilter implements WebFilter {
}
if (Objects.equals(path, indexPath) || "".equals(path)) {
if (request.getMethod() == HttpMethod.POST) {
ServerRequest serverRequest = ServerWebExchangeUtil.createReadBodyRequest(exchange);
ServerRequest serverRequest = ServerWebExchangeUtil.createReadBodyRequest(exchange, codecConfigurer);
// 读取请求体中的内容
Mono<?> modifiedBody = serverRequest.bodyToMono(byte[].class)
.switchIfEmpty(Mono.just("".getBytes()))

@ -5,9 +5,9 @@
<parent>
<groupId>com.gitee.sop</groupId>
<artifactId>sop-parent</artifactId>
<artifactId>sop-common</artifactId>
<version>${revision}</version>
<relativePath>../../pom.xml</relativePath> <!-- lookup parent from repository -->
<relativePath>../pom.xml</relativePath> <!-- lookup parent from repository -->
</parent>
<modelVersion>4.0.0</modelVersion>

@ -152,7 +152,7 @@ public class Example1001_BaseController {
// 忽略验证
@ApiOperation(value = "忽略签名验证", notes = "忽略签名验证")
@Open(value = "story.get.ignore", ignoreValidate = true)
@PostMapping(value = "/get/ignore/v1")
@RequestMapping(value = "/get/ignore/v1")
public StoryResult getStory21(@RequestBody StoryParam story) {
StoryResult result = new StoryResult();
result.setId((long) story.getId());

@ -5,6 +5,9 @@ import com.gitee.sop.gatewaycommon.interceptor.RouteInterceptorContext;
import com.gitee.sop.gatewaycommon.param.ApiParam;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.web.server.ServerWebExchange;
import java.net.URI;
/**
* 演示拦截器
@ -18,7 +21,10 @@ public class MyRouteInterceptor implements RouteInterceptor {
@Override
public void preRoute(RouteInterceptorContext context) {
ApiParam apiParam = context.getApiParam();
log.info("请求接口:{}, request_id:{}, app_id:{}, ip:{}",
ServerWebExchange exchange = (ServerWebExchange) context.getRequestContext();
URI uri = exchange.getRequest().getURI();
log.info("请求URL:{}, 请求接口:{}, request_id:{}, app_id:{}, ip:{}",
uri,
apiParam.fetchNameVersion(),
apiParam.fetchRequestId(),
apiParam.fetchAppKey(),

@ -34,6 +34,9 @@ spring.datasource.hikari.max-lifetime=500000
spring.cloud.gateway.discovery.locator.lower-case-service-id=true
spring.cloud.gateway.discovery.locator.enabled=true
# \u54CD\u5E94\u8D85\u65F6\uFF0C\u9ED8\u8BA410\u79D2\uFF0810000\uFF09
spring.cloud.gateway.httpclient.response-timeout=10000
spring.cloud.gateway.httpclient.pool.max-idle-time=60000
# \u4E0D\u7528\u6539
mybatis.mapper-locations=classpath:mybatis/mapper/*.xml

@ -12,7 +12,7 @@ register.url=http://localhost:1111/eureka/
# \u4E0A\u4F20\u6587\u4EF6\u6700\u5927\u5BB9\u91CF\uFF0C\u9ED8\u8BA410MB
spring.servlet.multipart.max-file-size=10MB
# \u54CD\u5E94\u8D85\u65F6\uFF0C\u9ED8\u8BA45\u79D2\uFF085000\uFF09
#spring.cloud.gateway.httpclient.response-timeout=5000
# \u54CD\u5E94\u8D85\u65F6\uFF0C\u9ED8\u8BA410\u79D2\uFF0810000\uFF09
#spring.cloud.gateway.httpclient.response-timeout=10000
logging.level.com.gitee=debug

@ -102,7 +102,7 @@ public class AllInOneTest extends TestBase {
.version("1.0")
.ignoreSign(true)
.bizContent(new BizContent().add("id", "222").add("name", "忽略222"))
.httpMethod(HttpTool.HTTPMethod.POST);
.httpMethod(HttpTool.HTTPMethod.GET);
client.execute(requestBuilder);
}

Loading…
Cancel
Save