优化文件上传校验

sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/ServerWebExchangeUtil.java

@@ -67,11 +67,13 @@ public class ServerWebExchangeUtil {
      * @return 没有参数返回null
      * @see com.gitee.sop.gatewaycommon.gateway.route.ReadBodyRoutePredicateFactory
      */
-    public static Map<String, ?> getRequestParams(ServerWebExchange exchange) {
-        Map<String, ?> params = exchange.getAttribute(CACHE_REQUEST_BODY_FOR_MAP);
-        if (params != null) {
-            return params;
+    public static ApiParam getRequestParams(ServerWebExchange exchange) {
+        ApiParam apiParamExist = exchange.getAttribute(CACHE_REQUEST_BODY_FOR_MAP);
+        if (apiParamExist != null) {
+            return apiParamExist;
         }
+        ApiParam apiParam = new ApiParam();
+        Map<String, ?> params = null;
         if (exchange.getRequest().getMethod() == HttpMethod.GET) {
             MultiValueMap<String, String> queryParams = exchange.getRequest().getQueryParams();
             params = buildParams(queryParams);
@@ -86,17 +88,19 @@ public class ServerWebExchangeUtil {
                 } else if (StringUtils.containsIgnoreCase(contentTypeStr, "multipart")) {
                     // 如果是文件上传请求
                     HttpServletRequest fileUploadRequest = getFileUploadRequest(exchange, cachedBody);
-                    setFileUploadRequest(exchange, fileUploadRequest);
-                    params = RequestUtil.convertMultipartRequestToMap(fileUploadRequest);
+                    RequestUtil.UploadInfo uploadInfo = RequestUtil.getUploadInfo(fileUploadRequest);
+                    params = uploadInfo.getUploadParams();
+                    apiParam.setUploadContext(uploadInfo.getUploadContext());
                 } else {
                     params = RequestUtil.parseQueryToMap(cachedBody);
                 }
             }
         }
         if (params != null) {
-            exchange.getAttributes().put(CACHE_REQUEST_BODY_FOR_MAP, params);
+            apiParam.putAll(params);
+            exchange.getAttributes().put(CACHE_REQUEST_BODY_FOR_MAP, apiParam);
         }
-        return params;
+        return apiParam;
     }
 
 

sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/param/GatewayParamBuilder.java

@@ -1,21 +1,18 @@
 package com.gitee.sop.gatewaycommon.gateway.param;
 
 import com.gitee.sop.gatewaycommon.gateway.ServerWebExchangeUtil;
+import com.gitee.sop.gatewaycommon.param.ApiParam;
 import com.gitee.sop.gatewaycommon.param.BaseParamBuilder;
 import org.springframework.web.server.ServerWebExchange;
 
-import java.util.Collections;
-import java.util.Map;
-
 /**
  * @author tanghc
  */
 public class GatewayParamBuilder extends BaseParamBuilder<ServerWebExchange> {
 
     @Override
-    public Map<String, ?> buildRequestParams(ServerWebExchange exchange) {
-        Map<String, ?> params = ServerWebExchangeUtil.getRequestParams(exchange);
-        return params == null ? Collections.emptyMap() : params;
+    public ApiParam buildRequestParams(ServerWebExchange exchange) {
+        return ServerWebExchangeUtil.getRequestParams(exchange);
     }
 
     @Override

sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/param/ApiParam.java

@@ -29,7 +29,7 @@ public class ApiParam extends JSONObject implements Param {
 
     private String ip;
 
-    private transient ApiUploadContext apiUploadContext;
+    private transient UploadContext uploadContext;
 
     public void fitNameVersion() {
         if (restName != null) {
@@ -213,12 +213,12 @@ public class ApiParam extends JSONObject implements Param {
         return getString(ParamNames.CHARSET_NAME);
     }
 
-    public void setApiUploadContext(ApiUploadContext apiUploadContext) {
-        this.apiUploadContext = apiUploadContext;
+    public void setUploadContext(UploadContext uploadContext) {
+        this.uploadContext = uploadContext;
     }
 
-    public ApiUploadContext fetchApiUploadContext() {
-        return apiUploadContext;
+    public UploadContext fetchUploadContext() {
+        return uploadContext;
     }
 
     @Override

sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/param/ApiUploadContext.java

@@ -14,6 +14,9 @@ import java.util.Map;
  */
 public class ApiUploadContext implements UploadContext {
 
+    /**
+     * key: 表单name
+     */
     private Map<String, MultipartFile> fileMap;
     private List<MultipartFile> allFile;
 

sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/param/BaseParamBuilder.java

@@ -8,7 +8,6 @@ import com.gitee.sop.gatewaycommon.message.ErrorEnum;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.BooleanUtils;
 
-import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -19,39 +18,41 @@ public abstract class BaseParamBuilder<T> implements ParamBuilder<T> {
 
     /**
      * 构建请求参数
+     *
      * @param ctx 请求request
      * @return 返回请求参数
      */
-    public abstract Map<String, ?> buildRequestParams(T ctx);
+    public abstract ApiParam buildRequestParams(T ctx);
 
     /**
      * 返回客户端ip
-     * @param ctx  请求request
+     *
+     * @param ctx 请求request
      * @return 返回ip
      */
     public abstract String getIP(T ctx);
 
     /**
      * 将版本号添加到header中
-     * @param ctx 请求request
+     *
+     * @param ctx        请求request
      * @param headerName headerName
-     * @param version 版本号
+     * @param version    版本号
      */
     public abstract void setVersionInHeader(T ctx, String headerName, String version);
 
     @Override
     public ApiParam build(T ctx) {
-        ApiParam apiParam = this.newApiParam(ctx);
-        Map<String, ?> requestParams = this.buildRequestParams(ctx);
-        apiParam.putAll(requestParams);
+        ApiParam apiParam = this.buildRequestParams(ctx);
+        this.processApiParam(apiParam, ctx);
         this.initOtherProperty(apiParam);
         apiParam.setIp(this.getIP(ctx));
         this.setVersionInHeader(ctx, ParamNames.HEADER_VERSION_NAME, apiParam.fetchVersion());
         return apiParam;
     }
 
-    protected ApiParam newApiParam(T ctx) {
-        return new ApiParam();
+    protected void processApiParam(ApiParam param, T ctx) {
+
     }
 
     protected void initOtherProperty(ApiParam apiParam) {

sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/util/RequestUtil.java

@@ -2,7 +2,10 @@ package com.gitee.sop.gatewaycommon.util;
 
 import com.alibaba.fastjson.JSON;
 import com.gitee.sop.gatewaycommon.bean.SopConstants;
+import com.gitee.sop.gatewaycommon.param.ApiUploadContext;
+import com.gitee.sop.gatewaycommon.param.UploadContext;
 import com.netflix.zuul.http.HttpServletRequestWrapper;
+import lombok.Data;
 import org.apache.commons.fileupload.FileItem;
 import org.apache.commons.fileupload.disk.DiskFileItemFactory;
 import org.apache.commons.fileupload.servlet.ServletFileUpload;
@@ -10,6 +13,8 @@ import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.multipart.commons.CommonsMultipartFile;
 
 import javax.servlet.http.HttpServletRequest;
 import java.io.IOException;
@@ -26,6 +31,8 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 
 /**
@@ -74,6 +81,7 @@ public class RequestUtil {
 
     /**
      * 将map参数转换成查询参数
+     *
      * @return 返回aa=1&b=c...
      */
     public static String convertMapToQueryString(Map<String, ?> apiParam) {
@@ -152,6 +160,7 @@ public class RequestUtil {
 
     /**
      * 转换json请求到Map，
+     *
      * @param request 请求类型为application/json的Request
      * @return 返回Map
      */
@@ -167,6 +176,7 @@ public class RequestUtil {
 
     /**
      * 是否是文件上传请求
+     *
      * @param request 请求
      * @return true：是
      */
@@ -184,6 +194,12 @@ public class RequestUtil {
         return IOUtils.toString(request.getInputStream(), UTF8);
     }
 
+    /**
+     * 获取客户端IP
+     *
+     * @param request request
+     * @return 返回ip
+     */
     public static String getIP(HttpServletRequest request) {
         String ipAddress = request.getHeader("x-forwarded-for");
         if (ipAddress == null || ipAddress.length() == 0 || IP_UNKNOWN.equalsIgnoreCase(ipAddress)) {
@@ -214,4 +230,48 @@ public class RequestUtil {
         }
         return ipAddress;
     }
+
+
+    /**
+     * 获取上传文件内容
+     *
+     * @param request request
+     * @return 返回文件内容和表单内容
+     */
+    public static UploadInfo getUploadInfo(HttpServletRequest request) {
+        UploadInfo uploadInfo = new UploadInfo();
+        // 创建一个文件上传解析器
+        ServletFileUpload upload = new ServletFileUpload(new DiskFileItemFactory());
+        Map<String, String> uploadParams = new HashMap<>(16);
+        UploadContext uploadContext = null;
+        try {
+            List<MultipartFile> multipartFileList = new ArrayList<>(16);
+            List<FileItem> fileItems = upload.parseRequest(request);
+            for (FileItem fileItem : fileItems) {
+                if (fileItem.isFormField()) {
+                    uploadParams.put(fileItem.getFieldName(), fileItem.getString(SopConstants.UTF8));
+                } else {
+                    multipartFileList.add(new CommonsMultipartFile(fileItem));
+                }
+            }
+            if (multipartFileList.size() > 0) {
+                Map<String, MultipartFile> multipartFileMap = multipartFileList
+                        .stream()
+                        .collect(Collectors.toMap(MultipartFile::getName, Function.identity()));
+                uploadContext = new ApiUploadContext(multipartFileMap);
+            }
+            uploadInfo.setUploadParams(uploadParams);
+            uploadInfo.setUploadContext(uploadContext);
+        } catch (Exception e) {
+            log.error("参数解析错误", e);
+        }
+        return uploadInfo;
+    }
+
+    @Data
+    public static class UploadInfo {
+        private Map<String, String> uploadParams;
+        private UploadContext uploadContext;
+    }
+
 }

sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/validate/ApiValidator.java

@@ -2,9 +2,9 @@ package com.gitee.sop.gatewaycommon.validate;
 
 import com.gitee.sop.gatewaycommon.bean.ApiConfig;
 import com.gitee.sop.gatewaycommon.bean.ApiContext;
-import com.gitee.sop.gatewaycommon.bean.RouteDefinition;
 import com.gitee.sop.gatewaycommon.bean.Isv;
 import com.gitee.sop.gatewaycommon.bean.RouteConfig;
+import com.gitee.sop.gatewaycommon.bean.RouteDefinition;
 import com.gitee.sop.gatewaycommon.bean.TargetRoute;
 import com.gitee.sop.gatewaycommon.manager.IPBlacklistManager;
 import com.gitee.sop.gatewaycommon.manager.IsvRoutePermissionManager;
@@ -20,7 +20,9 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.BooleanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.util.StringUtils;
+import org.springframework.util.unit.DataSize;
 import org.springframework.web.multipart.MultipartFile;
 
 import java.io.IOException;
@@ -56,6 +58,13 @@ public class ApiValidator implements Validator {
     @Autowired
     private RouteConfigManager routeConfigManager;
 
+    /**
+     * 单个文件大小
+     */
+    @Value("${upload.max-file-size:10MB}")
+    private String maxFileSize;
+
+
     @Override
     public void validate(ApiParam param) {
         checkIP(param);
@@ -78,6 +87,7 @@ public class ApiValidator implements Validator {
 
     /**
      * 是否在IP黑名单中
+     *
      * @param param 接口参数
      */
     protected void checkIP(ApiParam param) {
@@ -89,6 +99,7 @@ public class ApiValidator implements Validator {
 
     /**
      * 检测能否访问
+     *
      * @param param 接口参数
      */
     protected void checkEnable(ApiParam param) {
@@ -116,26 +127,42 @@ public class ApiValidator implements Validator {
      * @param param
      */
     protected void checkUploadFile(ApiParam param) {
-        UploadContext uploadContext = param.fetchApiUploadContext();
+        UploadContext uploadContext = param.fetchUploadContext();
         if (uploadContext != null) {
             try {
                 List<MultipartFile> files = uploadContext.getAllFile();
                 for (MultipartFile file : files) {
-                    // 客户端传来的文件md5
-                    String clientMd5 = param.getString(file.getName());
-                    if (clientMd5 != null) {
-                        String fileMd5 = DigestUtils.md5Hex(file.getBytes());
-                        if (!clientMd5.equals(fileMd5)) {
-                            throw ErrorEnum.ISV_UPLOAD_FAIL.getErrorMeta().getException();
-                        }
-                    }
+                    checkSingleFileSize(file);
+                    checkFileMd5(param, file);
                 }
             } catch (IOException e) {
                 log.error("验证上传文件MD5错误", e);
                 throw ErrorEnum.ISV_UPLOAD_FAIL.getErrorMeta().getException();
             }
         }
+    }
 
+    private void checkFileMd5(ApiParam param, MultipartFile file) throws IOException {
+        // 客户端传来的文件md5
+        String clientMd5 = param.getString(file.getName());
+        if (clientMd5 != null) {
+            String fileMd5 = DigestUtils.md5Hex(file.getBytes());
+            if (!clientMd5.equals(fileMd5)) {
+                throw ErrorEnum.ISV_UPLOAD_FAIL.getErrorMeta().getException();
+            }
+        }
+    }
+
+    /**
+     * 校验单个文件大小
+     *
+     * @param file 文件
+     */
+    private void checkSingleFileSize(MultipartFile file) {
+        long fileSize = file.getSize();
+        if (fileSize > DataSize.parse(maxFileSize).toBytes()) {
+            throw ErrorEnum.ISV_INVALID_FILE_SIZE.getErrorMeta().getException(file.getName(), maxFileSize);
+        }
     }
 
     protected void checkTimeout(ApiParam param) {
@@ -210,6 +237,7 @@ public class ApiValidator implements Validator {
 
     /**
      * 校验访问权限
+     *
      * @param apiParam
      */
     protected void checkPermission(ApiParam apiParam) {

sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/zuul/filter/PostResultFilter.java

@@ -5,11 +5,13 @@ import com.gitee.sop.gatewaycommon.bean.ApiContext;
 import com.gitee.sop.gatewaycommon.bean.SopConstants;
 import com.gitee.sop.gatewaycommon.result.ResultExecutor;
 import com.gitee.sop.gatewaycommon.zuul.RequestContextUtil;
+import com.netflix.util.Pair;
 import com.netflix.zuul.context.RequestContext;
 import com.netflix.zuul.exception.ZuulException;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.cloud.netflix.zuul.filters.support.FilterConstants;
+import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
 
 import javax.servlet.http.HttpServletResponse;
@@ -43,20 +45,31 @@ public class PostResultFilter extends BaseZuulFilter {
         if (StringUtils.containsIgnoreCase(contentType, MediaType.APPLICATION_OCTET_STREAM_VALUE)) {
             return null;
         }
-        InputStream responseDataStream = requestContext.getResponseDataStream();
         ApiConfig apiConfig = ApiContext.getApiConfig();
         ResultExecutor<RequestContext, String> resultExecutor = apiConfig.getZuulResultExecutor();
+        String serviceResult = getServiceResponseBody(requestContext);
+        String finalResult = resultExecutor.mergeResult(requestContext, serviceResult);
+        requestContext.setResponseBody(finalResult);
+        requestContext.getZuulResponseHeaders().add(new Pair<>(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE));
+        return null;
+    }
+
+    /**
+     * 获取微服务端返回的结果
+     *
+     * @param requestContext RequestContext
+     * @return 返回结果
+     */
+    private String getServiceResponseBody(RequestContext requestContext) {
         String serviceResult;
+        InputStream responseDataStream = requestContext.getResponseDataStream();
         try {
             serviceResult = IOUtils.toString(responseDataStream, SopConstants.CHARSET_UTF8);
         } catch (Exception e) {
             log.error("业务方无数据返回", e);
             serviceResult = SopConstants.EMPTY_JSON;
         }
-        String finalResult = resultExecutor.mergeResult(requestContext, serviceResult);
-        requestContext.setResponseBody(finalResult);
-        return null;
+        return serviceResult;
     }
 
-
 }

sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/zuul/param/ZuulParamBuilder.java

@@ -24,9 +24,10 @@ public class ZuulParamBuilder extends BaseParamBuilder<RequestContext> {
     private static final String GET = "get";
 
     @Override
-    public Map<String, ?> buildRequestParams(RequestContext ctx) {
+    public ApiParam buildRequestParams(RequestContext ctx) {
         HttpServletRequest request = ctx.getRequest();
         Map<String, ?> params;
+        ApiParam apiParam = new ApiParam();
         if (GET.equalsIgnoreCase(request.getMethod())) {
             params = RequestUtil.convertRequestParamsToMap(request);
         } else {
@@ -39,12 +40,15 @@ public class ZuulParamBuilder extends BaseParamBuilder<RequestContext> {
             if (StringUtils.containsAny(contentType, MediaType.APPLICATION_JSON_VALUE, MediaType.TEXT_PLAIN_VALUE)) {
                 params = RequestUtil.convertJsonRequestToMap(request);
             } else if (ServletFileUpload.isMultipartContent(request)) {
-                params = RequestUtil.convertMultipartRequestToMap(request);
+                RequestUtil.UploadInfo uploadInfo = RequestUtil.getUploadInfo(request);
+                params = uploadInfo.getUploadParams();
+                apiParam.setUploadContext(uploadInfo.getUploadContext());
             } else {
                 params = RequestUtil.convertRequestParamsToMap(request);
             }
         }
-        return params;
+        apiParam.putAll(params);
+        return apiParam;
     }
 
     @Override
@@ -58,14 +62,12 @@ public class ZuulParamBuilder extends BaseParamBuilder<RequestContext> {
     }
 
     @Override
-    protected ApiParam newApiParam(RequestContext ctx) {
-        ApiParam apiParam = super.newApiParam(ctx);
+    protected void processApiParam(ApiParam apiParam, RequestContext ctx) {
         HttpServletRequest request = ctx.getRequest();
         String method = (String) request.getAttribute(SopConstants.REDIRECT_METHOD_KEY);
         String version = (String) request.getAttribute(SopConstants.REDIRECT_VERSION_KEY);
         apiParam.setRestName(method);
         apiParam.setRestVersion(version);
-        return apiParam;
     }
 
 }

sop-common/sop-gateway-common/src/main/resources/i18n/open/error_en.properties

@@ -29,7 +29,7 @@ open.error_40002=Invalid parameter
 open.error_40002_isv.invalid-parameter=Parameter is invalid
 open.error_40002_isv.upload-fail=File upload failed
 open.error_40002_isv.invalid-file-extension=Invalid file extension
-open.error_40002_isv.invalid-file-size=Invalid file size
+open.error_40002_isv.invalid-file-size=Invalid {0} file size, the max size is {1}
 open.error_40002_isv.invalid-method=Nonexistent method name
 open.error_40002_isv.invalid-format=Invalid data format
 open.error_40002_isv.invalid-signature-type=Invalid signature type

sop-common/sop-gateway-common/src/main/resources/i18n/open/error_zh_CN.properties

@@ -27,7 +27,7 @@
 #open.error_40002_isv.invalid-parameter=参数无效
 #open.error_40002_isv.upload-fail=文件上传失败
 #open.error_40002_isv.invalid-file-extension=文件扩展名无效
-#open.error_40002_isv.invalid-file-size=文件大小无效
+#open.error_40002_isv.invalid-file-size={0}文件大小无效，单文件不得超过{1}
 #open.error_40002_isv.invalid-method=不存在的方法名
 #open.error_40002_isv.invalid-format=无效的数据格式
 #open.error_40002_isv.invalid-signature-type=无效的签名类型
@@ -84,7 +84,7 @@ open.error_40002=\u975e\u6cd5\u7684\u53c2\u6570
 open.error_40002_isv.invalid-parameter=\u53c2\u6570\u65e0\u6548
 open.error_40002_isv.upload-fail=\u6587\u4ef6\u4e0a\u4f20\u5931\u8d25
 open.error_40002_isv.invalid-file-extension=\u6587\u4ef6\u6269\u5c55\u540d\u65e0\u6548
-open.error_40002_isv.invalid-file-size=\u6587\u4ef6\u5927\u5c0f\u65e0\u6548
+open.error_40002_isv.invalid-file-size={0}\u6587\u4ef6\u5927\u5c0f\u65e0\u6548\uff0c\u5355\u6587\u4ef6\u4e0d\u5f97\u8d85\u8fc7{1}
 open.error_40002_isv.invalid-method=\u4e0d\u5b58\u5728\u7684\u65b9\u6cd5\u540d
 open.error_40002_isv.invalid-format=\u65e0\u6548\u7684\u6570\u636e\u683c\u5f0f
 open.error_40002_isv.invalid-signature-type=\u65e0\u6548\u7684\u7b7e\u540d\u7c7b\u578b

sop-gateway/src/main/resources/application-dev.properties

@@ -50,8 +50,11 @@ mybatis.fill.com.gitee.fastmybatis.core.support.DateFillUpdate=gmt_modified
 
 # 文件上传配置
 spring.servlet.multipart.enabled=true
-spring.servlet.multipart.max-file-size=20MB
-spring.servlet.multipart.max-request-size=20MB
+# 这里设置大一点没关系，真实大小由upload.max-file-size控制
+spring.servlet.multipart.max-file-size=50MB
+
+# 允许上传文件大小，不能超过这个值，单位：B,KB,MB
+upload.max-file-size=2MB
 
 # zipkin服务跟踪
 spring.zipkin.base-url=${zipkin.url}

sop-test/src/test/java/com/gitee/sop/test/AllInOneTest.java

@@ -197,6 +197,7 @@ public class AllInOneTest extends TestBase {
                 .addFile("file1", new File(root + "/src/main/resources/file1.txt"))
                 .addFile("file2", new File(root + "/src/main/resources/file2.txt"))
                 .callback((requestInfo, responseData) -> {
+                    System.out.println(responseData);
                     JSONObject jsonObject = JSON.parseObject(responseData);
                     JSONObject data = jsonObject.getJSONObject(requestInfo.getDataNode());
                     Assert.assertEquals(data.getString("code"), "10000");