diff --git a/changelog.md b/changelog.md index e29b698c..dbb4cebc 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,11 @@ # changelog +## 4.1.0 + +需要执行`sop-upgrade-4.1.0.sql` + +- 重构路由监控功能 + ## 4.0.3 - 可定义业务错误码（见`@Open`注解中的`bizCode`属性） diff --git a/doc/docs/files/10090_路由授权.md b/doc/docs/files/10090_路由授权.md index 30e88502..9823f89c 100644 --- a/doc/docs/files/10090_路由授权.md +++ b/doc/docs/files/10090_路由授权.md @@ -12,9 +12,9 @@ 默认情况下，接口访问时公开的，ISV都能访问。如果要设置某个接口访问权限，在`@Open`注解中指定permission=true。 如：`@Open(value = "permission.story.get", permission = true)`。这样该接口是需要经过授权给ISV才能访问的。 -重启服务后，登录admin，服务管理-路由列表界面中，操作操作列会出现一个授权按钮，点击出现授权窗口，勾选对应的角色即可完成授权。 +重启服务后，登录admin，服务管理-路由列表界面中，`访问权限`列会出现一个点击授权，点击出现授权窗口，勾选对应的角色即可完成授权。 -- 点击`授权`按钮，进行角色授权 +- `点击授权`，进行角色授权 ![admin预览](images/10090_1.png "10090_1.png") diff --git a/doc/docs/files/10093_路由监控.md b/doc/docs/files/10093_路由监控.md index b9ad73d2..b0e56a87 100644 --- a/doc/docs/files/10093_路由监控.md +++ b/doc/docs/files/10093_路由监控.md @@ -1,15 +1,6 @@ # 路由监控 -路由监控功能可以查看各个接口的调用情况，监控信息收集采用拦截器实现。 - -- 统计各个接口的调用次数、耗时等信息 -- 错误日志统一在网关负责收集 -- 只收集未知类型的错误日志，开发人员主动throw的异常不收集 -- 收集的日志存放在内存中，重启网关日志会消失 - -## 永久保存日志 - -默认收集的日志存放在内存中，重启网关日志会消失（见：`com.gitee.sop.gatewaycommon.monitor.MonitorManager.java`）。如果要永久保存日志内容，需要自己修改`MonitorManager` +路由监控功能可以查看各个接口的调用情况，监控信息收集采用拦截器实现，前往【服务管理】-【路由监控】查看 - 后台预览 @@ -17,6 +8,20 @@ ![监控日志](images/10093_2.png "10093_2.png") + +- 注意事项 + +处理完错误后，请及时`标记解决`，一个接口默认保存50条错误信息，采用LRU机制，淘汰老的。标记解决后则会空出一个位置存放新的错误信息。 + +重复的错误只会存放一条记录，然后累加错误次数，重复错误定义如下： + +`instanceId + routeId + errorMsg`，即一个实例 + 路由id + 错误信息确定一个错误 + +可在网关设置`sop.monitor.error-count-capacity=50`参数调整错误容量 + +考虑到数据库压力，网关收到错误信息后并不会立即保存到数据库，而是先保存到内容中，然后定时保存到时间，默认时间隔为30秒 +可通过`sop.monitor.flush-period-seconds=30`调整间隔时间。 + 相关类： -- com.gitee.sop.gatewaycommon.interceptor.MonitorRouteInterceptor +- com.gitee.sop.gateway.interceptor.MonitorRouteInterceptor diff --git a/doc/docs/files/images/10090_1.png b/doc/docs/files/images/10090_1.png index 493950c0..db577156 100644 Binary files a/doc/docs/files/images/10090_1.png and b/doc/docs/files/images/10090_1.png differ diff --git a/doc/docs/files/images/10090_2.png b/doc/docs/files/images/10090_2.png index ff3c76cf..3ec5b9b8 100644 Binary files a/doc/docs/files/images/10090_2.png and b/doc/docs/files/images/10090_2.png differ diff --git a/doc/docs/files/images/10093_1.png b/doc/docs/files/images/10093_1.png deleted file mode 100644 index e51da6f4..00000000 Binary files a/doc/docs/files/images/10093_1.png and /dev/null differ diff --git a/doc/docs/files/images/10093_2.png b/doc/docs/files/images/10093_2.png deleted file mode 100644 index b23cacde..00000000 Binary files a/doc/docs/files/images/10093_2.png and /dev/null differ diff --git a/pom.xml b/pom.xml index 1f8546a2..e761db9a 100644 --- a/pom.xml +++ b/pom.xml @@ -68,6 +68,7 @@ 2.9.2 1.16.9 6.2 + 5.2.0 @@ -202,6 +203,12 @@ ${asm.version} + + com.github.pagehelper + pagehelper + ${pagehelper.version} + + org.projectlombok lombok diff --git a/sop-admin/sop-admin-server/pom.xml b/sop-admin/sop-admin-server/pom.xml index 24c97fa0..da88be56 100644 --- a/sop-admin/sop-admin-server/pom.xml +++ b/sop-admin/sop-admin-server/pom.xml @@ -21,11 +21,6 @@ - - com.gitee.sop - sop-bridge-nacos - 4.0.3-SNAPSHOT - @@ -64,6 +59,11 @@ commons-lang3 + + com.github.pagehelper + pagehelper + + com.alibaba.nacos diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/MonitorNewApi.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/MonitorNewApi.java new file mode 100644 index 00000000..a4e564bc --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/MonitorNewApi.java @@ -0,0 +1,79 @@ +package com.gitee.sop.adminserver.api.service; + +import com.gitee.easyopen.annotation.Api; +import com.gitee.easyopen.annotation.ApiService; +import com.gitee.fastmybatis.core.query.Query; +import com.gitee.fastmybatis.core.query.Sort; +import com.gitee.fastmybatis.core.support.PageEasyui; +import com.gitee.fastmybatis.core.util.MapperUtil; +import com.gitee.sop.adminserver.api.service.param.InstanceMonitorSearchParam; +import com.gitee.sop.adminserver.api.service.param.MonitorErrorMsgParam; +import com.gitee.sop.adminserver.api.service.param.MonitorInfoErrorSolveParam; +import com.gitee.sop.adminserver.api.service.param.MonitorSearchParam; +import com.gitee.sop.adminserver.bean.RouteErrorCount; +import com.gitee.sop.adminserver.entity.MonitorInfoError; +import com.gitee.sop.adminserver.entity.MonitorSummary; +import com.gitee.sop.adminserver.mapper.MonitorInfoErrorMapper; +import com.gitee.sop.adminserver.mapper.MonitorInfoMapper; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author tanghc + */ +@ApiService +public class MonitorNewApi { + + @Autowired + private MonitorInfoMapper monitorInfoMapper; + + @Autowired + private MonitorInfoErrorMapper monitorInfoErrorMapper; + + @Api(name = "monitornew.data.page") + PageInfo listMonitor(MonitorSearchParam param) { + Query query = Query.build(param); + query.orderby("errorCount", Sort.DESC) + .orderby("avgTime", Sort.DESC); + return PageHelper.offsetPage(query.getStart(), query.getLimit()) + .doSelectPage(() -> monitorInfoMapper.listMonitorSummary(query)) + .toPageInfo(); + } + + @Api(name = "monitornew.routeid.data.get") + List listInstanceMonitor(InstanceMonitorSearchParam param) { + Query query = Query.build(param); + query.orderby("errorCount", Sort.DESC) + .orderby("avgTime", Sort.DESC); + return monitorInfoMapper.listInstanceMonitorInfo(query); + } + + private Map getRouteErrorCount() { + List routeErrorCounts = monitorInfoErrorMapper.listRouteErrorCount(); + return routeErrorCounts.stream() + .collect(Collectors.toMap(RouteErrorCount::getRouteId, RouteErrorCount::getCount)); + } + + @Api(name = "monitornew.error.page") + PageEasyui listError(MonitorErrorMsgParam param) { + Query query = param.toQuery() + .orderby("gmt_modified", Sort.DESC); + return MapperUtil.queryForEasyuiDatagrid(monitorInfoErrorMapper, query); + } + + @Api(name = "monitornew.error.solve") + void solve(MonitorInfoErrorSolveParam param) { + Query query = Query.build(param); + Map set = new HashMap<>(4); + set.put("is_deleted", 1); + set.put("count", 0); + monitorInfoErrorMapper.updateByMap(set, query); + } + +} diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/param/InstanceMonitorSearchParam.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/param/InstanceMonitorSearchParam.java new file mode 100644 index 00000000..a0545cde --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/param/InstanceMonitorSearchParam.java @@ -0,0 +1,15 @@ +package com.gitee.sop.adminserver.api.service.param; + +import com.gitee.fastmybatis.core.query.annotation.Condition; +import lombok.Getter; +import lombok.Setter; + +/** + * @author tanghc + */ +@Getter +@Setter +public class InstanceMonitorSearchParam { + @Condition(column = "t.route_id") + private String routeId; +} diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/param/MonitorErrorMsgParam.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/param/MonitorErrorMsgParam.java new file mode 100644 index 00000000..9df9d4b9 --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/param/MonitorErrorMsgParam.java @@ -0,0 +1,21 @@ +package com.gitee.sop.adminserver.api.service.param; + +import com.gitee.fastmybatis.core.query.annotation.Condition; +import com.gitee.fastmybatis.core.query.param.PageParam; +import lombok.Getter; +import lombok.Setter; + +import javax.validation.constraints.NotBlank; + +/** + * @author tanghc + */ +@Getter +@Setter +public class MonitorErrorMsgParam extends PageParam { + @NotBlank(message = "routeId不能为空") + private String routeId; + + @Condition(ignoreEmptyString = true) + private String instanceId; +} diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/param/MonitorInfoErrorSolveParam.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/param/MonitorInfoErrorSolveParam.java new file mode 100644 index 00000000..14a38950 --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/param/MonitorInfoErrorSolveParam.java @@ -0,0 +1,19 @@ +package com.gitee.sop.adminserver.api.service.param; + +import com.gitee.fastmybatis.core.query.annotation.Condition; +import lombok.Data; + +import javax.validation.constraints.NotBlank; + +/** + * @author tanghc + */ +@Data +public class MonitorInfoErrorSolveParam { + /** 错误id,md5(error_msg), 数据库字段：error_id */ + @NotBlank + @Condition(index = 1) + private String errorId; + + +} diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/param/MonitorSearchParam.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/param/MonitorSearchParam.java new file mode 100644 index 00000000..9d046140 --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/param/MonitorSearchParam.java @@ -0,0 +1,23 @@ +package com.gitee.sop.adminserver.api.service.param; + +import com.gitee.easyopen.doc.annotation.ApiDocField; +import com.gitee.fastmybatis.core.query.Operator; +import com.gitee.fastmybatis.core.query.annotation.Condition; +import com.gitee.fastmybatis.core.query.param.PageParam; +import lombok.Getter; +import lombok.Setter; + +/** + * @author tanghc + */ +@Getter +@Setter +public class MonitorSearchParam extends PageParam { + @ApiDocField(description = "服务名serviceId") + @Condition(column = "service_id", operator = Operator.like, ignoreEmptyString = true) + private String serviceId; + + @ApiDocField(description = "路由id") + @Condition(column = "route_id", operator = Operator.like, ignoreEmptyString = true) + private String routeId; +} diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/result/MonitorInfoErrorMsgResult.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/result/MonitorInfoErrorMsgResult.java new file mode 100644 index 00000000..7b85322b --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/api/service/result/MonitorInfoErrorMsgResult.java @@ -0,0 +1,26 @@ +package com.gitee.sop.adminserver.api.service.result; + +import lombok.Data; + +import java.util.Date; + +/** + * @author tanghc + */ +@Data +public class MonitorInfoErrorMsgResult { + + private String routeId; + + private String errorId; + + /** 错误信息, 数据库字段：error_msg */ + private String errorMsg; + + private Integer errorStatus; + + private Integer count; + + /** 数据库字段：gmt_modified */ + private Date gmtModified; +} diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/bean/RouteErrorCount.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/bean/RouteErrorCount.java new file mode 100644 index 00000000..b425504e --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/bean/RouteErrorCount.java @@ -0,0 +1,13 @@ +package com.gitee.sop.adminserver.bean; + +import lombok.Data; + +/** + * @author tanghc + */ +@Data +public class RouteErrorCount { + + private String routeId; + private Integer count; +} diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/common/CopyUtil.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/common/CopyUtil.java new file mode 100644 index 00000000..76c068fb --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/common/CopyUtil.java @@ -0,0 +1,75 @@ +package com.gitee.sop.adminserver.common; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * @author tanghc + */ +@Slf4j +public class CopyUtil { + + public static void copyProperties(Object from, Object to) { + BeanUtils.copyProperties(from, to); + } + + public static T copyBean(Object from, Supplier supplier) { + Objects.requireNonNull(from); + T to = supplier.get(); + BeanUtils.copyProperties(from, to); + return to; + } + + public static T copyBeanNullable(Object from, Supplier supplier) { + if (from == null) { + return supplier.get(); + } + T to = supplier.get(); + BeanUtils.copyProperties(from, to); + return to; + } + + public static T copyBean(Object from, Supplier supplier, Consumer after) { + if (from == null) { + return null; + } + T to = supplier.get(); + BeanUtils.copyProperties(from, to); + after.accept(to); + return to; + } + + public static List copyList(List fromList, Supplier toElement) { + if (fromList == null) { + return Collections.emptyList(); + } + return fromList.stream() + .map(source -> { + T target = toElement.get(); + BeanUtils.copyProperties(source, target); + return target; + }) + .collect(Collectors.toList()); + } + + public static List copyList(List fromList, Supplier toElement, Consumer after) { + if (fromList == null) { + return Collections.emptyList(); + } + return fromList.stream() + .map(source -> { + T target = toElement.get(); + BeanUtils.copyProperties(source, target); + after.accept(target); + return target; + }) + .collect(Collectors.toList()); + } +} diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/entity/MonitorInfo.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/entity/MonitorInfo.java new file mode 100644 index 00000000..5b721309 --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/entity/MonitorInfo.java @@ -0,0 +1,66 @@ +package com.gitee.sop.adminserver.entity; + +import lombok.Data; + +import javax.persistence.Column; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.util.Date; + + +/** + * 表名：monitor_info + * 备注：接口监控信息 + * + * @author tanghc + */ +@Table(name = "monitor_info") +@Data +public class MonitorInfo { + /** 数据库字段：id */ + @Id + @Column(name = "id") + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + /** 路由id, 数据库字段：route_id */ + private String routeId; + + /** 接口名, 数据库字段：name */ + private String name; + + /** 版本号, 数据库字段：version */ + private String version; + + /** 数据库字段：service_id */ + private String serviceId; + + /** 数据库字段：instance_id */ + private String instanceId; + + /** 请求耗时最长时间, 数据库字段：max_time */ + private Integer maxTime; + + /** 请求耗时最小时间, 数据库字段：min_time */ + private Integer minTime; + + /** 总时长，毫秒, 数据库字段：total_time */ + private Long totalTime; + + /** 总调用次数, 数据库字段：total_request_count */ + private Long totalRequestCount; + + /** 成功次数, 数据库字段：success_count */ + private Long successCount; + + /** 失败次数（业务主动抛出的异常算作成功，如参数校验，未知的错误算失败）, 数据库字段：error_count */ + private Long errorCount; + + /** 数据库字段：gmt_create */ + private Date gmtCreate; + + /** 数据库字段：gmt_modified */ + private Date gmtModified; +} diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/entity/MonitorInfoError.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/entity/MonitorInfoError.java new file mode 100644 index 00000000..69dcba9e --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/entity/MonitorInfoError.java @@ -0,0 +1,54 @@ +package com.gitee.sop.adminserver.entity; + +import lombok.Data; + +import javax.persistence.Column; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.util.Date; + + +/** + * 表名：monitor_info_error + * + * @author tanghc + */ +@Table(name = "monitor_info_error") +@Data +public class MonitorInfoError { + /** 数据库字段：id */ + @Id + @Column(name = "id") + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + /** 错误id,md5Hex(instanceId + routeId + errorMsg), 数据库字段：error_id */ + private String errorId; + + /** 实例id, 数据库字段：instance_id */ + private String instanceId; + + /** 数据库字段：route_id */ + private String routeId; + + /** 数据库字段：error_msg */ + private String errorMsg; + + /** http status，非200错误, 数据库字段：error_status */ + private Integer errorStatus; + + /** 错误次数, 数据库字段：count */ + private Integer count; + + /** 数据库字段：is_deleted */ + @com.gitee.fastmybatis.core.annotation.LogicDelete + private Byte isDeleted; + + /** 数据库字段：gmt_create */ + private Date gmtCreate; + + /** 数据库字段：gmt_modified */ + private Date gmtModified; +} diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/entity/MonitorSummary.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/entity/MonitorSummary.java new file mode 100644 index 00000000..3df26d02 --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/entity/MonitorSummary.java @@ -0,0 +1,61 @@ +package com.gitee.sop.adminserver.entity; + +import lombok.Data; + +import javax.persistence.Column; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + + +/** + * @author tanghc + */ +@Data +public class MonitorSummary { + + private static final AtomicInteger i = new AtomicInteger(); + + private Integer id = i.incrementAndGet(); + + private String routeId; + + private String name; + private String version; + + /** 数据库字段：service_id */ + private String serviceId; + + /** 数据库字段：instance_id */ + private String instanceId; + + /** 请求耗时最长时间, 数据库字段：max_time */ + private Integer maxTime; + + /** 请求耗时最小时间, 数据库字段：min_time */ + private Integer minTime; + + /** 总时长，毫秒, 数据库字段：total_time */ + private Long totalTime; + + /** 总调用次数, 数据库字段：total_request_count */ + private Long totalRequestCount; + + /** 成功次数, 数据库字段：success_count */ + private Long successCount; + + /** 失败次数（业务主动抛出的异常算作成功，如参数校验，未知的错误算失败）, 数据库字段：error_count */ + private Long errorCount; + + /** 未解决的错误数量 */ + private Long unsolvedErrorCount; + + private Float avgTime; + + private Boolean hasChildren; +} diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/mapper/MonitorInfoErrorMapper.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/mapper/MonitorInfoErrorMapper.java new file mode 100644 index 00000000..98a7be9d --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/mapper/MonitorInfoErrorMapper.java @@ -0,0 +1,22 @@ +package com.gitee.sop.adminserver.mapper; + +import com.gitee.fastmybatis.core.mapper.CrudMapper; +import com.gitee.sop.adminserver.bean.RouteErrorCount; +import com.gitee.sop.adminserver.entity.MonitorInfoError; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + + +/** + * @author tanghc + */ +public interface MonitorInfoErrorMapper extends CrudMapper { + + + @Select("SELECT route_id routeId, count(*) `count` FROM monitor_info_error \n" + + "WHERE is_deleted=0 \n" + + "GROUP BY route_id") + List listRouteErrorCount(); + +} diff --git a/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/mapper/MonitorInfoMapper.java b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/mapper/MonitorInfoMapper.java new file mode 100644 index 00000000..fc5de394 --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/java/com/gitee/sop/adminserver/mapper/MonitorInfoMapper.java @@ -0,0 +1,19 @@ +package com.gitee.sop.adminserver.mapper; + +import com.gitee.fastmybatis.core.mapper.CrudMapper; +import com.gitee.fastmybatis.core.query.Query; +import com.gitee.sop.adminserver.entity.MonitorInfo; +import com.gitee.sop.adminserver.entity.MonitorSummary; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + + +/** + * @author tanghc + */ +public interface MonitorInfoMapper extends CrudMapper { + List listMonitorSummary(@Param("query") Query query); + + List listInstanceMonitorInfo(@Param("query") Query query); +} diff --git a/sop-admin/sop-admin-server/src/main/resources/META-INF/sop-admin.properties b/sop-admin/sop-admin-server/src/main/resources/META-INF/sop-admin.properties index 50f585f0..f315ba05 100644 --- a/sop-admin/sop-admin-server/src/main/resources/META-INF/sop-admin.properties +++ b/sop-admin/sop-admin-server/src/main/resources/META-INF/sop-admin.properties @@ -30,6 +30,7 @@ spring.datasource.hikari.pool-name=HikariCP spring.datasource.hikari.max-lifetime=500000 # 固定不用改 +mybatis.config-location=classpath:mybatis/mybatisConfig.xml mybatis.mapper-locations=classpath:mybatis/mapper/*.xml easyopen.show-doc=false easyopen.ignore-validate=true diff --git a/sop-admin/sop-admin-server/src/main/resources/mybatis/mapper/MonitorInfoMapper.xml b/sop-admin/sop-admin-server/src/main/resources/mybatis/mapper/MonitorInfoMapper.xml new file mode 100644 index 00000000..2255cba6 --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/resources/mybatis/mapper/MonitorInfoMapper.xml @@ -0,0 +1,63 @@ + + + + + + + + + + + diff --git a/sop-admin/sop-admin-server/src/main/resources/mybatis/mybatisConfig.xml b/sop-admin/sop-admin-server/src/main/resources/mybatis/mybatisConfig.xml new file mode 100644 index 00000000..81ac0637 --- /dev/null +++ b/sop-admin/sop-admin-server/src/main/resources/mybatis/mybatisConfig.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/sop-admin/sop-admin-vue/src/router/index.js b/sop-admin/sop-admin-vue/src/router/index.js index 5476d797..990abc28 100644 --- a/sop-admin/sop-admin-vue/src/router/index.js +++ b/sop-admin/sop-admin-vue/src/router/index.js @@ -76,7 +76,7 @@ export const constantRoutes = [ { path: 'monitor', name: 'Monitor', - component: () => import('@/views/service/monitor'), + component: () => import('@/views/service/monitorNew'), meta: { title: '路由监控' } }, { diff --git a/sop-admin/sop-admin-vue/src/views/service/monitorNew.vue b/sop-admin/sop-admin-vue/src/views/service/monitorNew.vue new file mode 100644 index 00000000..5c1d57c8 --- /dev/null +++ b/sop-admin/sop-admin-vue/src/views/service/monitorNew.vue @@ -0,0 +1,273 @@ + + + diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/bean/LRUCache.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/bean/LRUCache.java new file mode 100644 index 00000000..9ea136d7 --- /dev/null +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/bean/LRUCache.java @@ -0,0 +1,43 @@ +package com.gitee.sop.gatewaycommon.bean; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.Function; + +/** + * LRU缓存 + * LinkedHashMap 本身内部有一个触发条件则自动执行的方法：删除最老元素（最近最少使用的元素） + * 由于最近最少使用元素是 LinkedHashMap 内部处理 + * 故我们不再需要维护 最近访问元素放在链尾，get 时直接访问/ put 时直接存储 + * created by Ethan-Walker on 2019/2/16 + */ +public class LRUCache { + private final Map map; + + public LRUCache(int capacity) { + map = new LinkedHashMap(capacity, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + // 容量大于capacity 时就删除 + return size() > capacity; + } + }; + } + public V get(K key) { + return map.get(key); + } + + public V computeIfAbsent(K key, + Function mappingFunction) { + return map.computeIfAbsent(key, mappingFunction); + } + + public V put(K key, V value) { + return map.put(key, value); + } + + public Collection values() { + return map.values(); + } +} \ No newline at end of file diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/AbstractConfiguration.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/AbstractConfiguration.java index 170db99b..e00b1d56 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/AbstractConfiguration.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/config/AbstractConfiguration.java @@ -5,7 +5,6 @@ import com.gitee.sop.gatewaycommon.bean.ApiContext; import com.gitee.sop.gatewaycommon.bean.BeanInitializer; import com.gitee.sop.gatewaycommon.bean.SpringContext; import com.gitee.sop.gatewaycommon.gateway.loadbalancer.NacosServerIntrospector; -import com.gitee.sop.gatewaycommon.interceptor.MonitorRouteInterceptor; import com.gitee.sop.gatewaycommon.interceptor.RouteInterceptor; import com.gitee.sop.gatewaycommon.limit.LimitManager; import com.gitee.sop.gatewaycommon.manager.EnvGrayManager; @@ -17,17 +16,20 @@ import com.gitee.sop.gatewaycommon.manager.LimitConfigManager; import com.gitee.sop.gatewaycommon.manager.RouteConfigManager; import com.gitee.sop.gatewaycommon.manager.RouteRepositoryContext; import com.gitee.sop.gatewaycommon.message.ErrorFactory; +import com.gitee.sop.gatewaycommon.monitor.MonitorManager; import com.gitee.sop.gatewaycommon.param.ParameterFormatter; import com.gitee.sop.gatewaycommon.route.RegistryListener; import com.gitee.sop.gatewaycommon.route.ServiceListener; import com.gitee.sop.gatewaycommon.route.ServiceRouteListener; import com.gitee.sop.gatewaycommon.secret.IsvManager; +import com.gitee.sop.gatewaycommon.sync.SopAsyncConfigurer; import com.gitee.sop.gatewaycommon.util.RouteInterceptorUtil; import com.gitee.sop.gatewaycommon.validate.SignConfig; import com.gitee.sop.gatewaycommon.validate.Validator; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -156,6 +158,17 @@ public class AbstractConfiguration implements ApplicationContextAware, Applicati return ApiConfig.getInstance().getParameterFormatter(); } + @Bean + public SopAsyncConfigurer sopAsyncConfigurer(@Value("${sop.monitor-route-interceptor.thread-pool-size:4}") + int threadPoolSize) { + return new SopAsyncConfigurer("gatewayAsync", threadPoolSize); + } + + @Bean + @ConditionalOnMissingBean + public MonitorManager monitorManager() { + return new MonitorManager(); + } /** * 跨域过滤器，gateway采用react形式，需要使用reactive包下的UrlBasedCorsConfigurationSource @@ -231,7 +244,6 @@ public class AbstractConfiguration implements ApplicationContextAware, Applicati protected void initRouteInterceptor() { Map routeInterceptorMap = applicationContext.getBeansOfType(RouteInterceptor.class); Collection routeInterceptors = new ArrayList<>(routeInterceptorMap.values()); - routeInterceptors.add(new MonitorRouteInterceptor()); RouteInterceptorUtil.addInterceptors(routeInterceptors); } diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/result/GatewayResultExecutor.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/result/GatewayResultExecutor.java index b745a3f5..65d9fe45 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/result/GatewayResultExecutor.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/gateway/result/GatewayResultExecutor.java @@ -39,7 +39,7 @@ public class GatewayResultExecutor extends BaseExecutorAdapter errorCodeList = exchange.getResponse().getHeaders().get(SopConstants.X_SERVICE_ERROR_CODE); if (!CollectionUtils.isEmpty(errorCodeList)) { String errorCode = errorCodeList.get(0); - responseStatus = Integer.valueOf(errorCode); + responseStatus = Integer.parseInt(errorCode); } return responseStatus; } diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/interceptor/MonitorRouteInterceptor.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/interceptor/MonitorRouteInterceptor.java deleted file mode 100644 index e2942743..00000000 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/interceptor/MonitorRouteInterceptor.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.gitee.sop.gatewaycommon.interceptor; - -import com.gitee.sop.gatewaycommon.bean.ApiConfig; -import com.gitee.sop.gatewaycommon.monitor.MonitorInfo; -import com.gitee.sop.gatewaycommon.monitor.MonitorManager; -import com.gitee.sop.gatewaycommon.param.ApiParam; - -import java.util.ArrayList; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -/** - * SOP默认的拦截器，用于收集监控数据 - * - * @author tanghc - */ -public class MonitorRouteInterceptor implements RouteInterceptor { - - private ThreadPoolExecutor threadPoolExecutor; - - public MonitorRouteInterceptor(int threadPoolSize) { - threadPoolExecutor = new ThreadPoolExecutor(threadPoolSize, threadPoolSize, - 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>()); - } - - public MonitorRouteInterceptor() { - this(4); - } - - public MonitorRouteInterceptor(ThreadPoolExecutor threadPoolExecutor) { - this.threadPoolExecutor = threadPoolExecutor; - } - - @Override - public void preRoute(RouteInterceptorContext context) { - } - - @Override - public void afterRoute(RouteInterceptorContext context) { - threadPoolExecutor.execute(() -> this.storeRequestInfo(context)); - } - - /** - * 记录接口调用流量，最大时间，最小时间，总时长，平均时长，调用次数，成功次数，失败次数. - * 需要考虑并发情况。 - */ - protected void storeRequestInfo(RouteInterceptorContext context) { - MonitorManager monitorManager = ApiConfig.getInstance().getMonitorManager(); - ApiParam apiParam = context.getApiParam(); - String routeId = apiParam.fetchNameVersion(); - long spendTime = context.getFinishTimeMillis() - context.getBeginTimeMillis(); - // 这步操作是线程安全的，底层调用了ConcurrentHashMap.computeIfAbsent - MonitorInfo monitorInfo = monitorManager.getMonitorInfo(routeId, (k) -> this.createMonitorInfo(apiParam)); - - monitorInfo.storeMaxTime(spendTime); - monitorInfo.storeMinTime(spendTime); - monitorInfo.getTotalCount().incrementAndGet(); - monitorInfo.getTotalTime().addAndGet(spendTime); - monitorInfo.getTotalRequestDataSize().addAndGet(context.getRequestDataSize()); - monitorInfo.getTotalResponseDataSize().addAndGet(context.getResponseDataSize()); - if (context.isSuccessRequest()) { - monitorInfo.getSuccessCount().incrementAndGet(); - } else { - monitorInfo.getErrorCount().incrementAndGet(); - String errorMsg = context.getServiceErrorMsg(); - monitorInfo.addErrorMsg(errorMsg); - } - } - - private MonitorInfo createMonitorInfo(ApiParam apiParam) { - MonitorInfo monitorInfo = new MonitorInfo(); - monitorInfo.setName(apiParam.fetchName()); - monitorInfo.setVersion(apiParam.fetchVersion()); - monitorInfo.setServiceId(apiParam.fetchServiceId()); - monitorInfo.setTotalRequestDataSize(new AtomicLong()); - monitorInfo.setTotalResponseDataSize(new AtomicLong()); - monitorInfo.setTotalTime(new AtomicLong()); - monitorInfo.setMaxTime(0L); - monitorInfo.setMinTime(0L); - monitorInfo.setSuccessCount(new AtomicLong()); - monitorInfo.setTotalCount(new AtomicLong()); - monitorInfo.setErrorCount(new AtomicLong()); - monitorInfo.setErrorMsgList(new ArrayList<>(10)); - return monitorInfo; - } - - - @Override - public int getOrder() { - return -1000; - } - -} diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/interceptor/RouteInterceptorContext.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/interceptor/RouteInterceptorContext.java index 9eaa8fcc..6e1d9dcf 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/interceptor/RouteInterceptorContext.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/interceptor/RouteInterceptorContext.java @@ -88,6 +88,6 @@ public interface RouteInterceptorContext { */ default boolean isSuccessRequest() { int responseStatus = getResponseStatus(); - return responseStatus == HttpStatus.OK.value() || responseStatus == SopConstants.BIZ_ERROR_STATUS; + return responseStatus == HttpStatus.OK.value(); } } diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/manager/EnvironmentKeys.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/manager/EnvironmentKeys.java index c5410563..8ba239cb 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/manager/EnvironmentKeys.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/manager/EnvironmentKeys.java @@ -33,8 +33,7 @@ public enum EnvironmentKeys { /** * post请求body缓存大小 */ - MAX_IN_MEMORY_SIZE("spring.codec.max-in-memory-size", "262144") - + MAX_IN_MEMORY_SIZE("spring.codec.max-in-memory-size", "262144"), ; diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorDTO.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorDTO.java new file mode 100644 index 00000000..2115c32e --- /dev/null +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorDTO.java @@ -0,0 +1,63 @@ +package com.gitee.sop.gatewaycommon.monitor; + +import lombok.Data; + +import java.util.Collection; + +/** + * 每个接口 总调用流量，最大时间，最小时间，总时长，平均时长，调用次数，成功次数，失败次数，错误查看。 + * + * @author tanghc + */ +@Data +public class MonitorDTO { + + /** + * 路由id + */ + private String routeId; + + /** + * 接口名 + */ + private String name; + /** + * 版本号 + */ + private String version; + /** + * serviceId + */ + private String serviceId; + + /** + * 实例id + */ + private String instanceId; + + /** 请求耗时最长时间, 数据库字段：max_time */ + private Integer maxTime; + + /** 请求耗时最小时间, 数据库字段：min_time */ + private Integer minTime; + + /** + * 总时长 + */ + private Long totalTime; + + /** 总调用次数, 数据库字段：total_request_count */ + private Long totalRequestCount; + + /** 成功次数, 数据库字段：success_count */ + private Long successCount; + + /** 失败次数（业务主动抛出的异常算作成功，如参数校验，未知的错误算失败）, 数据库字段：error_count */ + private Long errorCount; + + /** + * 错误信息 + */ + private Collection errorMsgList; + +} diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorData.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorData.java new file mode 100644 index 00000000..e957c873 --- /dev/null +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorData.java @@ -0,0 +1,106 @@ +package com.gitee.sop.gatewaycommon.monitor; + +import com.gitee.sop.gatewaycommon.bean.LRUCache; +import lombok.Data; +import org.apache.commons.codec.digest.DigestUtils; + +import java.time.LocalDateTime; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * 每个接口 总调用流量，最大时间，最小时间，总时长，平均时长，调用次数，成功次数，失败次数，错误查看。 + * + * @author tanghc + */ +@Data +public class MonitorData { + + public static final int LIMIT_SIZE = 50; + + + private String routeId; + + /** + * 接口名 + */ + private String name; + /** + * 版本号 + */ + private String version; + /** + * serviceId + */ + private String serviceId; + + private String instanceId; + + /** + * 请求耗时最长时间 + */ + private Integer maxTime; + /** + * 请求耗时最小时间 + */ + private Integer minTime; + /** + * 总时长 + */ + private AtomicInteger totalTime; + /** + * 总调用次数 + */ + private AtomicInteger totalRequestCount; + /** + * 成功次数 + */ + private AtomicInteger successCount; + /** + * 失败次数（业务主动抛出的异常算作成功，如参数校验，未知的错误算失败） + */ + private AtomicInteger errorCount; + + /** + * 错误信息,key: errorId + */ + private LRUCache monitorErrorMsgMap; + + /** + * 下一次刷新到数据库的时间 + */ + private LocalDateTime flushTime; + + public synchronized void storeMaxTime(int spendTime) { + if (spendTime > maxTime) { + maxTime = spendTime; + } + } + + public synchronized void storeMinTime(int spendTime) { + if (minTime == 0 || spendTime < minTime) { + minTime = spendTime; + } + } + + public void addErrorMsg(String errorMsg, int httpStatus) { + if (errorMsg == null || "".equals(errorMsg)) { + return; + } + synchronized (this) { + String errorId = DigestUtils.md5Hex(instanceId + routeId + errorMsg); + MonitorErrorMsg monitorErrorMsg = monitorErrorMsgMap.computeIfAbsent(errorId, (k) -> { + MonitorErrorMsg value = new MonitorErrorMsg(); + value.setErrorId(errorId); + value.setInstanceId(instanceId); + value.setRouteId(routeId); + value.setErrorMsg(errorMsg); + value.setErrorStatus(httpStatus); + value.setCount(0); + return value; + }); + monitorErrorMsg.setCount(monitorErrorMsg.getCount() + 1); + } + } + +} diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorErrorMsg.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorErrorMsg.java new file mode 100644 index 00000000..a14799da --- /dev/null +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorErrorMsg.java @@ -0,0 +1,30 @@ +package com.gitee.sop.gatewaycommon.monitor; + +import lombok.Data; + +/** + * @author thc + */ +@Data +public class MonitorErrorMsg { + /** 错误id,md5(error_msg), 数据库字段：error_id */ + private String errorId; + + /** 实例id, 数据库字段：instance_id */ + private String instanceId; + + /** 数据库字段：route_id */ + private String routeId; + + /** 数据库字段：isp_id */ + private Long ispId; + + /** 数据库字段：error_msg */ + private String errorMsg; + + /** http status，非200错误 */ + private Integer errorStatus; + + /** 错误次数, 数据库字段：count */ + private Integer count; +} \ No newline at end of file diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorInfo.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorInfo.java deleted file mode 100644 index a05bd56a..00000000 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorInfo.java +++ /dev/null @@ -1,88 +0,0 @@ -package com.gitee.sop.gatewaycommon.monitor; - -import lombok.Data; - -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -/** - * 每个接口 总调用流量，最大时间，最小时间，总时长，平均时长，调用次数，成功次数，失败次数，错误查看。 - * - * @author tanghc - */ -@Data -public class MonitorInfo { - - /** - * 接口名 - */ - private String name; - /** - * 版本号 - */ - private String version; - /** - * serviceId - */ - private String serviceId; - /** - * 总请求数据量 - */ - private AtomicLong totalRequestDataSize; - /** - * 总返回数据量 - */ - private AtomicLong totalResponseDataSize; - /** - * 请求耗时最长时间 - */ - private Long maxTime; - /** - * 请求耗时最小时间 - */ - private Long minTime; - /** - * 总时长 - */ - private AtomicLong totalTime; - /** - * 总调用次数 - */ - private AtomicLong totalCount; - /** - * 成功次数 - */ - private AtomicLong successCount; - /** - * 失败次数（业务主动抛出的异常算作成功，如参数校验，未知的错误算失败） - */ - private AtomicLong errorCount; - /** - * 错误信息 - */ - private List errorMsgList; - - public synchronized void storeMaxTime(long spendTime) { - if (spendTime > maxTime) { - maxTime = spendTime; - } - } - - public synchronized void storeMinTime(long spendTime) { - if (minTime == 0 || spendTime < minTime) { - minTime = spendTime; - } - } - - public void addErrorMsg(String errorMsg) { - if (errorMsg == null || "".equals(errorMsg)) { - return; - } - synchronized (this) { - if (errorMsgList != null && errorMsgList.size() < 10) { - errorMsgList.add(errorMsg); - } - } - } - -} diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorManager.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorManager.java index a181da7e..15865dea 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorManager.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/MonitorManager.java @@ -9,13 +9,13 @@ import java.util.function.Function; */ public class MonitorManager { - private static Map monitorMap = new ConcurrentHashMap<>(128); + private static Map monitorMap = new ConcurrentHashMap<>(128); - public Map getMonitorData() { + public Map getMonitorData() { return monitorMap; } - public MonitorInfo getMonitorInfo(String routeId, Function createFun) { + public MonitorData getMonitorInfo(String routeId, Function createFun) { return monitorMap.computeIfAbsent(routeId, createFun); } diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/RouteErrorCount.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/RouteErrorCount.java new file mode 100644 index 00000000..2ff46699 --- /dev/null +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/monitor/RouteErrorCount.java @@ -0,0 +1,13 @@ +package com.gitee.sop.gatewaycommon.monitor; + +import lombok.Data; + +/** + * @author tanghc + */ +@Data +public class RouteErrorCount { + + private String routeId; + private Integer count; +} diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java index f3b3f329..76c68aec 100644 --- a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/result/BaseExecutorAdapter.java @@ -119,7 +119,7 @@ public abstract class BaseExecutorAdapter implements ResultExecutor defaultRouteInterceptorContext.setServiceResult(serviceResult); defaultRouteInterceptorContext.setFinishTimeMillis(System.currentTimeMillis()); defaultRouteInterceptorContext.setResponseDataSize(serviceResult.length()); - if (responseStatus != HttpStatus.OK.value() && responseStatus != SopConstants.BIZ_ERROR_STATUS) { + if (responseStatus != HttpStatus.OK.value()) { String responseErrorMessage = getResponseErrorMessage(requestContext); if (StringUtils.isEmpty(responseErrorMessage)) { responseErrorMessage = serviceResult; diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/sync/MyNamedThreadFactory.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/sync/MyNamedThreadFactory.java new file mode 100644 index 00000000..267af1d1 --- /dev/null +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/sync/MyNamedThreadFactory.java @@ -0,0 +1,37 @@ +package com.gitee.sop.gatewaycommon.sync; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author thc + */ +public class MyNamedThreadFactory implements ThreadFactory { + private static final AtomicInteger POOL_NUMBER = new AtomicInteger(1); + private final AtomicInteger threadNumber = new AtomicInteger(1); + private final ThreadGroup group; + private final String namePrefix; + + public MyNamedThreadFactory(String name) { + + SecurityManager s = System.getSecurityManager(); + group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup(); + if (null == name || name.isEmpty()) { + name = "pool"; + } + + namePrefix = name + "-" + POOL_NUMBER.getAndIncrement() + "-thread-"; + } + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement()); + if (t.isDaemon()) { + t.setDaemon(false); + } + if (t.getPriority() != Thread.NORM_PRIORITY) { + t.setPriority(Thread.NORM_PRIORITY); + } + return t; + } +} \ No newline at end of file diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/sync/SopAsyncConfigurer.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/sync/SopAsyncConfigurer.java new file mode 100644 index 00000000..eb609c4e --- /dev/null +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/sync/SopAsyncConfigurer.java @@ -0,0 +1,41 @@ +package com.gitee.sop.gatewaycommon.sync; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler; +import org.springframework.scheduling.annotation.AsyncConfigurer; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * 异步执行配置 + * + * @author tanghc + */ +@Slf4j +public class SopAsyncConfigurer implements AsyncConfigurer { + + private final ThreadPoolExecutor threadPoolExecutor; + + public SopAsyncConfigurer(String threadName, int poolSize) { + threadPoolExecutor = new ThreadPoolExecutor(poolSize, poolSize, + 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), new MyNamedThreadFactory(threadName)); + } + + @Override + public Executor getAsyncExecutor() { + return threadPoolExecutor; + } + + @Override + public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() { + return (Throwable e, Method method, Object... args) -> { + log.error("异步运行方法出错, method:{}, args:{}, message:{}", method, Arrays.deepToString(args), e.getMessage(), e); + }; + } +} diff --git a/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/util/CopyUtil.java b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/util/CopyUtil.java new file mode 100644 index 00000000..724ca63f --- /dev/null +++ b/sop-common/sop-gateway-common/src/main/java/com/gitee/sop/gatewaycommon/util/CopyUtil.java @@ -0,0 +1,79 @@ +package com.gitee.sop.gatewaycommon.util; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * @author tanghc + */ +@Slf4j +public class CopyUtil { + + public static void copyPropertiesIgnoreNull(Object from, Object to) { + MyBeanUtil.copyPropertiesIgnoreNull(from, to); + } + + public static void copyProperties(Object from, Object to) { + BeanUtils.copyProperties(from, to); + } + + public static T copyBean(Object from, Supplier supplier) { + Objects.requireNonNull(from); + T to = supplier.get(); + BeanUtils.copyProperties(from, to); + return to; + } + + public static T copyBeanNullable(Object from, Supplier supplier) { + if (from == null) { + return supplier.get(); + } + T to = supplier.get(); + BeanUtils.copyProperties(from, to); + return to; + } + + public static T copyBean(Object from, Supplier supplier, Consumer after) { + if (from == null) { + return null; + } + T to = supplier.get(); + BeanUtils.copyProperties(from, to); + after.accept(to); + return to; + } + + public static List copyList(List fromList, Supplier toElement) { + if (fromList == null) { + return Collections.emptyList(); + } + return fromList.stream() + .map(source -> { + T target = toElement.get(); + BeanUtils.copyProperties(source, target); + return target; + }) + .collect(Collectors.toList()); + } + + public static List copyList(List fromList, Supplier toElement, Consumer after) { + if (fromList == null) { + return Collections.emptyList(); + } + return fromList.stream() + .map(source -> { + T target = toElement.get(); + BeanUtils.copyProperties(source, target); + after.accept(target); + return target; + }) + .collect(Collectors.toList()); + } +} diff --git a/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/configuration/GlobalExceptionHandler.java b/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/configuration/GlobalExceptionHandler.java index 610fc271..7507a6a0 100644 --- a/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/configuration/GlobalExceptionHandler.java +++ b/sop-common/sop-service-common/src/main/java/com/gitee/sop/servercommon/configuration/GlobalExceptionHandler.java @@ -72,7 +72,7 @@ public class GlobalExceptionHandler { int lineCount = 5; for (int i = 0; i < stackTrace.length && i < lineCount; i++) { StackTraceElement stackTraceElement = stackTrace[i]; - msg.append("
at ").append(stackTraceElement.toString()); + msg.append("\n at ").append(stackTraceElement.toString()); } response.setHeader("x-service-error-message", UriUtils.encode(msg.toString(), StandardCharsets.UTF_8)); return this.processError(request, response, new ServiceException("系统繁忙")); diff --git a/sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java b/sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java index 238e2370..73393281 100644 --- a/sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java +++ b/sop-example/sop-story/src/main/java/com/gitee/sop/storyweb/controller/Example1001_BaseController.java @@ -18,6 +18,7 @@ import com.gitee.sop.storyweb.controller.result.TreeResult; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; @@ -50,6 +51,9 @@ import java.util.Map; @Api(tags = "故事接口") public class Example1001_BaseController { + @Value("${server.port}") + private int port; + // http://localhost:2222/stroy/get // 原生的接口，可正常调用 @RequestMapping("/get") @@ -71,7 +75,7 @@ public class Example1001_BaseController { public StoryResult get_v1(StoryParam param) { StoryResult story = new StoryResult(); story.setId(1L); - story.setName("海底小纵队(story.get1.0), " + "param:" + param); + story.setName("海底小纵队(story.get1.0), " + "param:" + param + ", port:" + port); return story; } diff --git a/sop-gateway/src/main/java/com/gitee/sop/gateway/entity/MonitorInfo.java b/sop-gateway/src/main/java/com/gitee/sop/gateway/entity/MonitorInfo.java new file mode 100644 index 00000000..a73b77f6 --- /dev/null +++ b/sop-gateway/src/main/java/com/gitee/sop/gateway/entity/MonitorInfo.java @@ -0,0 +1,67 @@ +package com.gitee.sop.gateway.entity; + +import lombok.Data; + +import javax.persistence.Column; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.time.LocalDateTime; +import java.util.Date; + + +/** + * 表名：monitor_info + * 备注：接口监控信息 + * + * @author tanghc + */ +@Table(name = "monitor_info") +@Data +public class MonitorInfo { + /** 数据库字段：id */ + @Id + @Column(name = "id") + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + /** 路由id, 数据库字段：route_id */ + private String routeId; + + /** 接口名, 数据库字段：name */ + private String name; + + /** 版本号, 数据库字段：version */ + private String version; + + /** 数据库字段：service_id */ + private String serviceId; + + /** 数据库字段：instance_id */ + private String instanceId; + + /** 请求耗时最长时间, 数据库字段：max_time */ + private Integer maxTime; + + /** 请求耗时最小时间, 数据库字段：min_time */ + private Integer minTime; + + /** 总时长，毫秒, 数据库字段：total_time */ + private Long totalTime; + + /** 总调用次数, 数据库字段：total_request_count */ + private Long totalRequestCount; + + /** 成功次数, 数据库字段：success_count */ + private Long successCount; + + /** 失败次数（业务主动抛出的异常算作成功，如参数校验，未知的错误算失败）, 数据库字段：error_count */ + private Long errorCount; + + /** 数据库字段：gmt_create */ + private Date gmtCreate; + + /** 数据库字段：gmt_modified */ + private Date gmtModified; +} diff --git a/sop-gateway/src/main/java/com/gitee/sop/gateway/entity/MonitorInfoError.java b/sop-gateway/src/main/java/com/gitee/sop/gateway/entity/MonitorInfoError.java new file mode 100644 index 00000000..1927770a --- /dev/null +++ b/sop-gateway/src/main/java/com/gitee/sop/gateway/entity/MonitorInfoError.java @@ -0,0 +1,55 @@ +package com.gitee.sop.gateway.entity; + +import lombok.Data; + +import javax.persistence.Column; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.time.LocalDateTime; +import java.util.Date; + + +/** + * 表名：monitor_info_error + * + * @author tanghc + */ +@Table(name = "monitor_info_error") +@Data +public class MonitorInfoError { + /** 数据库字段：id */ + @Id + @Column(name = "id") + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + /** 错误id,md5Hex(instanceId + routeId + errorMsg), 数据库字段：error_id */ + private String errorId; + + /** 实例id, 数据库字段：instance_id */ + private String instanceId; + + /** 数据库字段：route_id */ + private String routeId; + + /** 数据库字段：error_msg */ + private String errorMsg; + + /** http status，非200错误, 数据库字段：error_status */ + private Integer errorStatus; + + /** 错误次数, 数据库字段：count */ + private Integer count; + + /** 数据库字段：is_deleted */ + @com.gitee.fastmybatis.core.annotation.LogicDelete + private Byte isDeleted; + + /** 数据库字段：gmt_create */ + private Date gmtCreate; + + /** 数据库字段：gmt_modified */ + private Date gmtModified; +} diff --git a/sop-gateway/src/main/java/com/gitee/sop/gateway/interceptor/MonitorRouteInterceptor.java b/sop-gateway/src/main/java/com/gitee/sop/gateway/interceptor/MonitorRouteInterceptor.java new file mode 100644 index 00000000..7d4d28b3 --- /dev/null +++ b/sop-gateway/src/main/java/com/gitee/sop/gateway/interceptor/MonitorRouteInterceptor.java @@ -0,0 +1,43 @@ +package com.gitee.sop.gateway.interceptor; + +import com.gitee.sop.gatewaycommon.interceptor.RouteInterceptor; +import com.gitee.sop.gatewaycommon.interceptor.RouteInterceptorContext; +import com.gitee.sop.gatewaycommon.param.ApiParam; +import com.gitee.sop.gatewaycommon.sync.SopAsyncConfigurer; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * 用于收集监控数据 + * + * @author tanghc + */ +@Component +@Slf4j +public class MonitorRouteInterceptor implements RouteInterceptor { + + @Autowired + SopAsyncConfigurer sopAsyncConfigurer; + + @Autowired + MonitorRouteInterceptorService monitorRouteInterceptorService; + + @Override + public void preRoute(RouteInterceptorContext context) { + } + + @Override + public void afterRoute(RouteInterceptorContext context) { + sopAsyncConfigurer.getAsyncExecutor().execute(()-> { + monitorRouteInterceptorService.storeRequestInfo(context); + }); + } + + @Override + public int getOrder() { + return -1000; + } + + +} diff --git a/sop-gateway/src/main/java/com/gitee/sop/gateway/interceptor/MonitorRouteInterceptorService.java b/sop-gateway/src/main/java/com/gitee/sop/gateway/interceptor/MonitorRouteInterceptorService.java new file mode 100644 index 00000000..1188ca6c --- /dev/null +++ b/sop-gateway/src/main/java/com/gitee/sop/gateway/interceptor/MonitorRouteInterceptorService.java @@ -0,0 +1,164 @@ +package com.gitee.sop.gateway.interceptor; + +import com.gitee.sop.gateway.mapper.DbMonitorInfoManager; +import com.gitee.sop.gatewaycommon.bean.LRUCache; +import com.gitee.sop.gatewaycommon.interceptor.RouteInterceptorContext; +import com.gitee.sop.gatewaycommon.monitor.MonitorDTO; +import com.gitee.sop.gatewaycommon.monitor.MonitorData; +import com.gitee.sop.gatewaycommon.monitor.MonitorManager; +import com.gitee.sop.gatewaycommon.param.ApiParam; +import com.gitee.sop.gatewaycommon.sync.MyNamedThreadFactory; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author tanghc + */ +@Service +@Slf4j +public class MonitorRouteInterceptorService { + + /** + * 刷新到数据库时间间隔，秒 + */ + @Value("${sop.monitor.flush-period-seconds:30}") + int flushPeriodSeconds; + + /** + * 定时任务每n秒，执行一次 + */ + @Value("${sop.monitor.schedule-period-seconds:30}") + int schedulePeriodSeconds; + + /** + * 错误数量容量 + */ + @Value("${sop.monitor.error-count-capacity:50}") + int monitorErrorCapacity; + + @Autowired + DbMonitorInfoManager dbMonitorInfoManager; + + @Autowired + MonitorManager monitorManager; + + /** + * 记录接口调用流量，最大时间，最小时间，总时长，平均时长，调用次数，成功次数，失败次数. + * 需要考虑并发情况。 + */ + public synchronized void storeRequestInfo(RouteInterceptorContext context) { + ApiParam apiParam = context.getApiParam(); + ServiceInstance serviceInstance = context.getServiceInstance(); + String routeId = apiParam.getRouteId(); + int spendTime = (int)(context.getFinishTimeMillis() - context.getBeginTimeMillis()); + // 这步操作是线程安全的，底层调用了ConcurrentHashMap.computeIfAbsent + String key = getMonitorKey(routeId, serviceInstance.getInstanceId()); + MonitorData monitorData = monitorManager.getMonitorInfo(key, (k) -> this.createMonitorInfo(apiParam, serviceInstance)); + monitorData.storeMaxTime(spendTime); + monitorData.storeMinTime(spendTime); + monitorData.getTotalRequestCount().incrementAndGet(); + monitorData.getTotalTime().addAndGet(spendTime); + if (context.isSuccessRequest()) { + monitorData.getSuccessCount().incrementAndGet(); + } else { + monitorData.getErrorCount().incrementAndGet(); + String errorMsg = context.getServiceErrorMsg(); + monitorData.addErrorMsg(errorMsg, context.getResponseStatus()); + } + } + + private String getMonitorKey(String routeId, String instanceId) { + return routeId + instanceId; + } + + /** + * 刷新到数据库 + */ + private synchronized void flushDb() { + Map monitorData = monitorManager.getMonitorData(); + if (monitorData.isEmpty()) { + return; + } + LocalDateTime checkTime = LocalDateTime.now(); + List tobeRemoveKeys = new ArrayList<>(); + List tobeSaveBatch = new ArrayList<>(monitorData.size()); + monitorData.forEach((key, value) -> { + LocalDateTime flushTime = value.getFlushTime(); + if (flushTime.isEqual(checkTime) || flushTime.isBefore(checkTime)) { + log.debug("刷新监控数据到数据库, MonitorData:{}", value); + tobeRemoveKeys.add(key); + MonitorDTO monitorDTO = getMonitorDTO(value); + tobeSaveBatch.add(monitorDTO); + } + }); + dbMonitorInfoManager.saveMonitorInfoBatch(tobeSaveBatch); + + for (String key : tobeRemoveKeys) { + monitorData.remove(key); + } + } + + private MonitorDTO getMonitorDTO(MonitorData monitorData) { + MonitorDTO monitorDTO = new MonitorDTO(); + monitorDTO.setRouteId(monitorData.getRouteId()); + monitorDTO.setName(monitorData.getName()); + monitorDTO.setVersion(monitorData.getVersion()); + monitorDTO.setServiceId(monitorData.getServiceId()); + monitorDTO.setInstanceId(monitorData.getInstanceId()); + monitorDTO.setMaxTime(monitorData.getMaxTime()); + monitorDTO.setMinTime(monitorData.getMinTime()); + monitorDTO.setTotalTime(monitorData.getTotalTime().longValue()); + monitorDTO.setTotalRequestCount(monitorData.getTotalRequestCount().longValue()); + monitorDTO.setSuccessCount(monitorData.getSuccessCount().longValue()); + monitorDTO.setErrorCount(monitorData.getErrorCount().longValue()); + monitorDTO.setErrorMsgList(monitorData.getMonitorErrorMsgMap().values()); + return monitorDTO; + } + + private MonitorData createMonitorInfo(ApiParam apiParam, ServiceInstance serviceInstance) { + MonitorData monitorData = new MonitorData(); + monitorData.setRouteId(apiParam.getRouteId()); + monitorData.setName(apiParam.fetchName()); + monitorData.setVersion(apiParam.fetchVersion()); + monitorData.setServiceId(apiParam.fetchServiceId()); + monitorData.setInstanceId(serviceInstance.getInstanceId()); + monitorData.setTotalTime(new AtomicInteger()); + monitorData.setMaxTime(0); + monitorData.setMinTime(0); + monitorData.setSuccessCount(new AtomicInteger()); + monitorData.setTotalRequestCount(new AtomicInteger()); + monitorData.setErrorCount(new AtomicInteger()); + monitorData.setFlushTime(getFlushTime()); + monitorData.setMonitorErrorMsgMap(new LRUCache<>(monitorErrorCapacity)); + return monitorData; + } + + private LocalDateTime getFlushTime() { + return LocalDateTime.now() + .plusSeconds(flushPeriodSeconds); + } + + + @PostConstruct + public void after() { + // 每隔schedulePeriodSeconds秒执行一次 + ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(1, new MyNamedThreadFactory("monitorSchedule")); + // 延迟执行，随机5~14秒 + int delay = 5 + new Random().nextInt(10); + scheduledThreadPoolExecutor.scheduleWithFixedDelay(this::flushDb, delay, schedulePeriodSeconds, TimeUnit.SECONDS); + + } +} diff --git a/sop-gateway/src/main/java/com/gitee/sop/gateway/mapper/DbMonitorInfoManager.java b/sop-gateway/src/main/java/com/gitee/sop/gateway/mapper/DbMonitorInfoManager.java new file mode 100644 index 00000000..aa206b51 --- /dev/null +++ b/sop-gateway/src/main/java/com/gitee/sop/gateway/mapper/DbMonitorInfoManager.java @@ -0,0 +1,62 @@ +package com.gitee.sop.gateway.mapper; + +import com.gitee.sop.gatewaycommon.monitor.MonitorDTO; +import com.gitee.sop.gatewaycommon.monitor.MonitorErrorMsg; +import com.gitee.sop.gatewaycommon.monitor.RouteErrorCount; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * @author tanghc + */ +@Service +public class DbMonitorInfoManager { + + @Autowired + private MonitorInfoMapper monitorInfoMapper; + + @Autowired + private MonitorInfoErrorMapper monitorInfoErrorMapper; + + @Value("${sop.monitor.error-count-capacity:50}") + int limitCount; + + public void saveMonitorInfoBatch(List list) { + if (CollectionUtils.isEmpty(list)) { + return; + } + monitorInfoMapper.saveMonitorInfoBatch(list); + this.saveMonitorInfoErrorBatch(list); + } + + private void saveMonitorInfoErrorBatch(List list) { + List routeErrorCounts = monitorInfoErrorMapper.listRouteErrorCountAll(); + // 路由id对应的错误次数，key：routeId，value：错误次数 + Map routeErrorCountsMap = routeErrorCounts.stream() + .collect(Collectors.toMap(RouteErrorCount::getRouteId, RouteErrorCount::getCount)); + List monitorErrorMsgList = list.stream() + .filter(monitorDTO -> CollectionUtils.isNotEmpty(monitorDTO.getErrorMsgList())) + .flatMap(monitorDTO -> { + int limit = limitCount - routeErrorCountsMap.getOrDefault(monitorDTO.getRouteId(), 0); + // 容量已满 + if (limit <= 0) { + return null; + } + // 截取剩余 + return monitorDTO.getErrorMsgList().stream().limit(limit); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(monitorErrorMsgList)) { + monitorInfoErrorMapper.saveMonitorInfoErrorBatch(monitorErrorMsgList); + } + } + +} diff --git a/sop-gateway/src/main/java/com/gitee/sop/gateway/mapper/MonitorInfoErrorMapper.java b/sop-gateway/src/main/java/com/gitee/sop/gateway/mapper/MonitorInfoErrorMapper.java new file mode 100644 index 00000000..c58dc524 --- /dev/null +++ b/sop-gateway/src/main/java/com/gitee/sop/gateway/mapper/MonitorInfoErrorMapper.java @@ -0,0 +1,36 @@ +package com.gitee.sop.gateway.mapper; + +import com.gitee.fastmybatis.core.mapper.CrudMapper; +import com.gitee.sop.gateway.entity.MonitorInfoError; +import com.gitee.sop.gatewaycommon.monitor.MonitorErrorMsg; +import com.gitee.sop.gatewaycommon.monitor.RouteErrorCount; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +import java.util.List; + + +/** + * @author tanghc + */ +public interface MonitorInfoErrorMapper extends CrudMapper { + + @Update("UPDATE monitor_info_error " + + "SET is_deleted=0 " + + ",count=count + 1 " + + "WHERE route_id=#{routeId} AND error_id=#{errorId}") + int updateError(@Param("routeId") String routeId,@Param("errorId") String errorId); + + int saveMonitorInfoErrorBatch(@Param("list") List list); + + @Select("SELECT route_id routeId, count(*) `count` FROM monitor_info_error \n" + + "WHERE is_deleted=0 \n" + + "GROUP BY route_id") + List listRouteErrorCount(); + + @Select("SELECT route_id routeId, count(*) `count` FROM monitor_info_error \n" + + "WHERE is_deleted=0 \n" + + "GROUP BY route_id") + List listRouteErrorCountAll(); +} diff --git a/sop-gateway/src/main/java/com/gitee/sop/gateway/mapper/MonitorInfoMapper.java b/sop-gateway/src/main/java/com/gitee/sop/gateway/mapper/MonitorInfoMapper.java new file mode 100644 index 00000000..5d171f89 --- /dev/null +++ b/sop-gateway/src/main/java/com/gitee/sop/gateway/mapper/MonitorInfoMapper.java @@ -0,0 +1,39 @@ +package com.gitee.sop.gateway.mapper; + +import com.gitee.fastmybatis.core.mapper.CrudMapper; +import com.gitee.sop.gateway.entity.MonitorInfo; +import com.gitee.sop.gatewaycommon.monitor.MonitorDTO; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Update; + +import java.util.List; + + +/** + * @author tanghc + */ +public interface MonitorInfoMapper extends CrudMapper { + + /** + * 更新监控状态 + * + * @return 返回影响行数 + */ + @Update("UPDATE monitor_info " + + "set max_time=case when max_time < #{maxTime} then #{maxTime} else max_time end " + + ",min_time=case when min_time > #{minTime} then #{minTime} else min_time end " + + ",total_request_count=total_request_count + #{totalRequestCount} " + + ",total_time=total_time + #{totalTime} " + + ",success_count=success_count + #{successCount} " + + ",error_count=error_count + #{errorCount} " + + "where route_id=#{routeId}") + int updateMonitorInfo(MonitorDTO monitorDTO); + + + /** + * 批量插入监控数据 + * @param list 监控数据 + * @return 返回影响行数 + */ + int saveMonitorInfoBatch(@Param("list") List list); +} diff --git a/sop-gateway/src/main/resources/META-INF/gateway.properties b/sop-gateway/src/main/resources/META-INF/gateway.properties index 332006c1..99eca1a0 100644 --- a/sop-gateway/src/main/resources/META-INF/gateway.properties +++ b/sop-gateway/src/main/resources/META-INF/gateway.properties @@ -29,6 +29,7 @@ spring.cloud.gateway.discovery.locator.lower-case-service-id=true spring.cloud.gateway.discovery.locator.enabled=true # 不用改 +mybatis.mapper-locations=classpath:mybatis/mapper/*.xml mybatis.fill.com.gitee.fastmybatis.core.support.DateFillInsert=gmt_create mybatis.fill.com.gitee.fastmybatis.core.support.DateFillUpdate=gmt_modified diff --git a/sop-gateway/src/main/resources/mybatis/mapper/MonitorInfoErrorMapper.xml b/sop-gateway/src/main/resources/mybatis/mapper/MonitorInfoErrorMapper.xml new file mode 100644 index 00000000..1655177b --- /dev/null +++ b/sop-gateway/src/main/resources/mybatis/mapper/MonitorInfoErrorMapper.xml @@ -0,0 +1,30 @@ + + + + + + + INSERT INTO `monitor_info_error` ( + `error_id`, + `instance_id`, + `route_id`, + `error_msg`, + `error_status`, + `count`) + VALUES + + (#{data.errorId}, + #{data.instanceId}, + #{data.routeId}, + #{data.errorMsg}, + #{data.errorStatus}, + #{data.count}) + + ON DUPLICATE KEY UPDATE + error_msg = VALUES(error_msg) + , error_status = VALUES(error_status) + , `count`= `count` + VALUES(count) + , is_deleted = 0 + + + diff --git a/sop-gateway/src/main/resources/mybatis/mapper/MonitorInfoMapper.xml b/sop-gateway/src/main/resources/mybatis/mapper/MonitorInfoMapper.xml new file mode 100644 index 00000000..b485dbf0 --- /dev/null +++ b/sop-gateway/src/main/resources/mybatis/mapper/MonitorInfoMapper.xml @@ -0,0 +1,43 @@ + + + + + + + INSERT INTO `monitor_info` ( + `route_id`, + `name`, + `version`, + `service_id`, + `instance_id`, + `max_time`, + `min_time`, + `total_time`, + `total_request_count`, + `success_count`, + `error_count`) + VALUES + + (#{data.routeId}, + #{data.name}, + #{data.version}, + #{data.serviceId}, + #{data.instanceId}, + #{data.maxTime}, + #{data.minTime}, + #{data.totalTime}, + #{data.totalRequestCount}, + #{data.successCount}, + #{data.errorCount}) + + VALUES(min_time) then VALUES(min_time) else min_time end + ,total_time = total_time + VALUES(total_time) + ,total_request_count = total_request_count + VALUES(total_request_count) + ,success_count = success_count + VALUES(success_count) + ,error_count = error_count + VALUES(error_count) + ]]> + + diff --git a/sop-mysql5.6以下版本.sql b/sop-mysql5.6以下版本.sql index 8e02579a..0b497846 100644 --- a/sop-mysql5.6以下版本.sql +++ b/sop-mysql5.6以下版本.sql @@ -17,6 +17,8 @@ DROP TABLE IF EXISTS `config_gray`; DROP TABLE IF EXISTS `config_common`; DROP TABLE IF EXISTS `admin_user_info`; DROP TABLE IF EXISTS `config_service_route`; +DROP TABLE IF EXISTS `monitor_info`; +DROP TABLE IF EXISTS `monitor_info_error`; CREATE TABLE `admin_user_info` ( @@ -208,6 +210,40 @@ CREATE TABLE `config_service_route` ( KEY `idx_serviceid` (`service_id`) USING BTREE ) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='路由配置'; +CREATE TABLE `monitor_info` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `route_id` varchar(128) NOT NULL DEFAULT '' COMMENT '路由id', + `name` varchar(128) NOT NULL DEFAULT '' COMMENT '接口名', + `version` varchar(64) NOT NULL DEFAULT '' COMMENT '版本号', + `service_id` varchar(64) NOT NULL DEFAULT '', + `instance_id` varchar(128) NOT NULL DEFAULT '', + `max_time` int(11) NOT NULL DEFAULT '0' COMMENT '请求耗时最长时间', + `min_time` int(11) NOT NULL DEFAULT '0' COMMENT '请求耗时最小时间', + `total_time` bigint(20) NOT NULL DEFAULT '0' COMMENT '总时长，毫秒', + `total_request_count` bigint(20) NOT NULL DEFAULT '0' COMMENT '总调用次数', + `success_count` bigint(20) NOT NULL DEFAULT '0' COMMENT '成功次数', + `error_count` bigint(20) NOT NULL DEFAULT '0' COMMENT '失败次数（业务主动抛出的异常算作成功，如参数校验，未知的错误算失败）', + `gmt_create` DATETIME DEFAULT NULL, + `gmt_modified` DATETIME DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_routeid` (`route_id`) USING BTREE, + KEY `idex_name` (`name`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='接口监控信息'; + +CREATE TABLE `monitor_info_error` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `error_id` varchar(64) NOT NULL DEFAULT '' COMMENT '错误id,md5Hex(instanceId + routeId + errorMsg)', + `route_id` varchar(128) NOT NULL DEFAULT '', + `error_msg` text NOT NULL, + `error_status` int(11) NOT NULL DEFAULT '0' COMMENT 'http status，非200错误', + `count` int(11) NOT NULL DEFAULT '0' COMMENT '错误次数', + `is_deleted` tinyint(4) NOT NULL DEFAULT '0', + `gmt_create` DATETIME DEFAULT NULL, + `gmt_modified` DATETIME DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_erririd` (`error_id`) USING BTREE, + KEY `idx_routeid` (`route_id`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; INSERT INTO `admin_user_info` (`id`, `username`, `password`, `status`, `gmt_create`, `gmt_modified`) VALUES (1,'admin','a62cd510fb9a8a557a27ef279569091f',1,'2019-04-02 19:55:26','2019-04-02 19:55:26'); diff --git a/sop-upgrade-4.1.0.sql b/sop-upgrade-4.1.0.sql new file mode 100644 index 00000000..ed08f526 --- /dev/null +++ b/sop-upgrade-4.1.0.sql @@ -0,0 +1,38 @@ +-- 4.1.0升级脚本 + +use sop; + +CREATE TABLE `monitor_info` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `route_id` varchar(128) NOT NULL DEFAULT '' COMMENT '路由id', + `name` varchar(128) NOT NULL DEFAULT '' COMMENT '接口名', + `version` varchar(64) NOT NULL DEFAULT '' COMMENT '版本号', + `service_id` varchar(64) NOT NULL DEFAULT '', + `instance_id` varchar(128) NOT NULL DEFAULT '', + `max_time` int(11) NOT NULL DEFAULT '0' COMMENT '请求耗时最长时间', + `min_time` int(11) NOT NULL DEFAULT '0' COMMENT '请求耗时最小时间', + `total_time` bigint(20) NOT NULL DEFAULT '0' COMMENT '总时长，毫秒', + `total_request_count` bigint(20) NOT NULL DEFAULT '0' COMMENT '总调用次数', + `success_count` bigint(20) NOT NULL DEFAULT '0' COMMENT '成功次数', + `error_count` bigint(20) NOT NULL DEFAULT '0' COMMENT '失败次数（业务主动抛出的异常算作成功，如参数校验，未知的错误算失败）', + `gmt_create` datetime DEFAULT CURRENT_TIMESTAMP, + `gmt_modified` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_routeid` (`route_id`) USING BTREE, + KEY `idex_name` (`name`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='接口监控信息'; + +CREATE TABLE `monitor_info_error` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `error_id` varchar(64) NOT NULL DEFAULT '' COMMENT '错误id,md5Hex(instanceId + routeId + errorMsg)', + `route_id` varchar(128) NOT NULL DEFAULT '', + `error_msg` text NOT NULL, + `error_status` int(11) NOT NULL DEFAULT '0' COMMENT 'http status，非200错误', + `count` int(11) NOT NULL DEFAULT '0' COMMENT '错误次数', + `is_deleted` tinyint(4) NOT NULL DEFAULT '0', + `gmt_create` datetime DEFAULT CURRENT_TIMESTAMP, + `gmt_modified` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_erririd` (`error_id`) USING BTREE, + KEY `idx_routeid` (`route_id`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file diff --git a/sop.sql b/sop.sql index 3b44a9e7..73c70030 100644 --- a/sop.sql +++ b/sop.sql @@ -19,6 +19,8 @@ DROP TABLE IF EXISTS `config_gray_instance`; DROP TABLE IF EXISTS `config_gray`; DROP TABLE IF EXISTS `config_common`; DROP TABLE IF EXISTS `admin_user_info`; +DROP TABLE IF EXISTS `monitor_info`; +DROP TABLE IF EXISTS `monitor_info_error`; CREATE TABLE `admin_user_info` ( @@ -210,6 +212,40 @@ CREATE TABLE `user_info` ( KEY `idx_unamepwd` (`username`,`password`) USING BTREE ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COMMENT='用户信息表'; +CREATE TABLE `monitor_info` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `route_id` varchar(128) NOT NULL DEFAULT '' COMMENT '路由id', + `name` varchar(128) NOT NULL DEFAULT '' COMMENT '接口名', + `version` varchar(64) NOT NULL DEFAULT '' COMMENT '版本号', + `service_id` varchar(64) NOT NULL DEFAULT '', + `instance_id` varchar(128) NOT NULL DEFAULT '', + `max_time` int(11) NOT NULL DEFAULT '0' COMMENT '请求耗时最长时间', + `min_time` int(11) NOT NULL DEFAULT '0' COMMENT '请求耗时最小时间', + `total_time` bigint(20) NOT NULL DEFAULT '0' COMMENT '总时长，毫秒', + `total_request_count` bigint(20) NOT NULL DEFAULT '0' COMMENT '总调用次数', + `success_count` bigint(20) NOT NULL DEFAULT '0' COMMENT '成功次数', + `error_count` bigint(20) NOT NULL DEFAULT '0' COMMENT '失败次数（业务主动抛出的异常算作成功，如参数校验，未知的错误算失败）', + `gmt_create` datetime DEFAULT CURRENT_TIMESTAMP, + `gmt_modified` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_routeid` (`route_id`) USING BTREE, + KEY `idex_name` (`name`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='接口监控信息'; + +CREATE TABLE `monitor_info_error` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `error_id` varchar(64) NOT NULL DEFAULT '' COMMENT '错误id,md5Hex(instanceId + routeId + errorMsg)', + `route_id` varchar(128) NOT NULL DEFAULT '', + `error_msg` text NOT NULL, + `error_status` int(11) NOT NULL DEFAULT '0' COMMENT 'http status，非200错误', + `count` int(11) NOT NULL DEFAULT '0' COMMENT '错误次数', + `is_deleted` tinyint(4) NOT NULL DEFAULT '0', + `gmt_create` datetime DEFAULT CURRENT_TIMESTAMP, + `gmt_modified` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_erririd` (`error_id`) USING BTREE, + KEY `idx_routeid` (`route_id`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; INSERT INTO `admin_user_info` (`id`, `username`, `password`, `status`, `gmt_create`, `gmt_modified`) VALUES (1,'admin','a62cd510fb9a8a557a27ef279569091f',1,'2019-04-02 19:55:26','2019-04-02 19:55:26');