From f8bb30669baf93c5c2272e06311ca0faf403d6e2 Mon Sep 17 00:00:00 2001 From: Hoshi <1196756653@qq.com> Date: Tue, 20 Aug 2024 17:57:29 +0800 Subject: [PATCH] Initial commit all --- README.md | 1 + pom.xml | 400 ++++++++++++++++++ .../com/bocloud/cms/entity/MonitorAuth.java | 23 + .../com/bocloud/ims/entity/CloudVendor.java | 207 +++++++++ .../java/com/bocloud/ims/entity/KvmHost.java | 25 ++ .../com/bocloud/ims/enums/CloudProvider.java | 17 + .../java/com/bocloud/mcs/Application.java | 21 + .../com/bocloud/mcs/aspect/AccessAspect.java | 53 +++ .../bocloud/mcs/aspect/EnableCollector.java | 24 ++ .../mcs/collector/BocloudCollector.java | 87 ++++ .../mcs/collector/kvm/ConnectHolder.java | 44 ++ .../mcs/collector/kvm/KvmHostCollector.java | 116 +++++ .../mcs/collector/kvm/KvmVmCollector.java | 112 +++++ .../mcs/collector/kvm/vm/KvmVmAdapter.java | 17 + .../collector/kvm/vm/KvmVmCpuMetricData.java | 44 ++ .../collector/kvm/vm/KvmVmDiskMetricData.java | 65 +++ .../kvm/vm/KvmVmInterfacesMetricData.java | 70 +++ .../kvm/vm/KvmVmMemoryMetricData.java | 51 +++ .../manageone/ManageOneEvsDiskCollector.java | 105 +++++ .../manageone/ManageOneHostCollector.java | 110 +++++ .../manageone/ManageOneVmCollector.java | 110 +++++ .../collector/manageone/bms/BmsCollector.java | 138 ++++++ .../manageone/bms/common/ManageCommon.java | 70 +++ .../bms/common/ManageOneBmsClient.java | 104 +++++ .../collector/manageone/cce/CceCollector.java | 142 +++++++ .../cce/common/ContainerCluster.java | 160 +++++++ .../manageone/cce/common/ManageOneAuth.java | 63 +++ .../cce/common/ManageOneCatalog.java | 91 ++++ .../cce/common/ManageOneCceClient.java | 187 ++++++++ .../manageone/cce/common/ManageOneCommon.java | 70 +++ .../manageone/cce/common/ManageOneDomain.java | 62 +++ .../cce/common/ManageOneEndpoint.java | 89 ++++ .../cce/common/ManageOneIdentity.java | 57 +++ .../cce/common/ManageOnePassword.java | 37 ++ .../cce/common/ManageOneProject.java | 87 ++++ .../cce/common/ManageOneResponse.java | 21 + .../manageone/cce/common/ManageOneRole.java | 53 +++ .../manageone/cce/common/ManageOneScope.java | 37 ++ .../manageone/cce/common/ManageOneToken.java | 181 ++++++++ .../manageone/cce/common/ManageOneUser.java | 89 ++++ .../cce/common/ManagerMonitorEntity.java | 89 ++++ .../manageone/core/ManageOneClient.java | 294 +++++++++++++ .../tianyicloud/TianyiCloudCollector.java | 113 +++++ .../tianyicloud/core/MessageDigestUtils.java | 147 +++++++ .../tianyicloud/core/TianyiHeaderUtils.java | 126 ++++++ .../tianyicloud/core/TianyiProvider.java | 206 +++++++++ .../bocloud/mcs/common/Authentication.java | 62 +++ .../mcs/config/CollectorConfiguration.java | 35 ++ .../bocloud/mcs/config/ScheduledConfig.java | 24 ++ .../config/WebInterceptorConfiguration.java | 29 ++ .../mcs/controller/MetricController.java | 45 ++ .../mcs/controller/PrometheusController.java | 50 +++ .../mcs/entity/PrometheusHttpConfigBean.java | 27 ++ .../internal/CmpCloudVendorService.java | 78 ++++ .../internal/CmsCloudVendorService.java | 54 +++ .../internal/ImsCloudServerService.java | 68 +++ .../mcs/syslog/SysLogAppenderTcpAndUdp.java | 15 + .../mcs/syslog/SysLogLevelConvertor.java | 22 + .../syslog/SyslogOutputStreamTcpAndUdp.java | 70 +++ src/main/resources/application.yml | 111 +++++ src/main/resources/banner.txt | 11 + src/main/resources/collector.conf | 20 + src/main/resources/logback-spring.xml | 75 ++++ 63 files changed, 5211 insertions(+) create mode 100644 README.md create mode 100644 pom.xml create mode 100644 src/main/java/com/bocloud/cms/entity/MonitorAuth.java create mode 100644 src/main/java/com/bocloud/ims/entity/CloudVendor.java create mode 100644 src/main/java/com/bocloud/ims/entity/KvmHost.java create mode 100644 src/main/java/com/bocloud/ims/enums/CloudProvider.java create mode 100644 src/main/java/com/bocloud/mcs/Application.java create mode 100644 src/main/java/com/bocloud/mcs/aspect/AccessAspect.java create mode 100644 src/main/java/com/bocloud/mcs/aspect/EnableCollector.java create mode 100644 src/main/java/com/bocloud/mcs/collector/BocloudCollector.java create mode 100644 src/main/java/com/bocloud/mcs/collector/kvm/ConnectHolder.java create mode 100644 src/main/java/com/bocloud/mcs/collector/kvm/KvmHostCollector.java create mode 100644 src/main/java/com/bocloud/mcs/collector/kvm/KvmVmCollector.java create mode 100644 src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmAdapter.java create mode 100644 src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmCpuMetricData.java create mode 100644 src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmDiskMetricData.java create mode 100644 src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmInterfacesMetricData.java create mode 100644 src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmMemoryMetricData.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/ManageOneEvsDiskCollector.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/ManageOneHostCollector.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/ManageOneVmCollector.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/bms/BmsCollector.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/bms/common/ManageCommon.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/bms/common/ManageOneBmsClient.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/CceCollector.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ContainerCluster.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneAuth.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneCatalog.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneCceClient.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneCommon.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneDomain.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneEndpoint.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneIdentity.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOnePassword.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneProject.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneResponse.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneRole.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneScope.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneToken.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneUser.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManagerMonitorEntity.java create mode 100644 src/main/java/com/bocloud/mcs/collector/manageone/core/ManageOneClient.java create mode 100644 src/main/java/com/bocloud/mcs/collector/tianyicloud/TianyiCloudCollector.java create mode 100644 src/main/java/com/bocloud/mcs/collector/tianyicloud/core/MessageDigestUtils.java create mode 100644 src/main/java/com/bocloud/mcs/collector/tianyicloud/core/TianyiHeaderUtils.java create mode 100644 src/main/java/com/bocloud/mcs/collector/tianyicloud/core/TianyiProvider.java create mode 100644 src/main/java/com/bocloud/mcs/common/Authentication.java create mode 100644 src/main/java/com/bocloud/mcs/config/CollectorConfiguration.java create mode 100644 src/main/java/com/bocloud/mcs/config/ScheduledConfig.java create mode 100644 src/main/java/com/bocloud/mcs/config/WebInterceptorConfiguration.java create mode 100644 src/main/java/com/bocloud/mcs/controller/MetricController.java create mode 100644 src/main/java/com/bocloud/mcs/controller/PrometheusController.java create mode 100644 src/main/java/com/bocloud/mcs/entity/PrometheusHttpConfigBean.java create mode 100644 src/main/java/com/bocloud/mcs/service/internal/CmpCloudVendorService.java create mode 100644 src/main/java/com/bocloud/mcs/service/internal/CmsCloudVendorService.java create mode 100644 src/main/java/com/bocloud/mcs/service/internal/ImsCloudServerService.java create mode 100644 src/main/java/com/bocloud/mcs/syslog/SysLogAppenderTcpAndUdp.java create mode 100644 src/main/java/com/bocloud/mcs/syslog/SysLogLevelConvertor.java create mode 100644 src/main/java/com/bocloud/mcs/syslog/SyslogOutputStreamTcpAndUdp.java create mode 100644 src/main/resources/application.yml create mode 100644 src/main/resources/banner.txt create mode 100644 src/main/resources/collector.conf create mode 100644 src/main/resources/logback-spring.xml diff --git a/README.md b/README.md new file mode 100644 index 0000000..cd3fe1b --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +Metrics Collection Service \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..1f2d7fe --- /dev/null +++ b/pom.xml @@ -0,0 +1,400 @@ + + + + com.freedom + megatron + 3.0.0 + + 4.0.0 + com.bocloud + bocloud.mcs.booter + ${bocloud.booter.version} + bocloud.mcs + BoCloud Exporter + + 21 + 6.5.0-LTS + 1.7 + 0.6.0 + 3.2.0 + 11.2.0.3 + 3.0.30-rc + 6.5.1 + 1.2.17 + 2.5.11 + 2.1.1 + 1.0.1 + 1.2.0 + 0.10.192 + 2.15.35 + 4.9.2 + + + + + com.ksyun + ksc-sdk-java-bom + 0.3.31 + pom + import + + + + + + + org.slf4j + slf4j-api + 2.0.7 + + + org.apache.httpcomponents.client5 + httpclient5 + + + jdk.tools + jdk.tools + 1.6 + + + + com.zstack + zstack.sdk + 4.4.0 + + + com.smartx.tower + cloudtower-java-sdk + 2.8.0 + + + org.threeten + threetenbp + 1.5.0 + + + io.gsonfire + gson-fire + 1.8.3 + + + com.squareup.okhttp3 + okhttp + ${okhttp-version} + + + com.squareup.okhttp3 + logging-interceptor + ${okhttp-version} + + + + + org.libvirt + libvirt + 0.5.3.bocloud + + + + org.jsoup + jsoup + 1.13.1 + + + com.freedom + megatron.microservice + 3.0.0 + + + com.freedom + megatron.framework + + + com.freedom + megatron.common + + + com.bocloud + bocloud.boot.common + ${bocloud.boot.common.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + compile + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + ${jackson.version} + compile + + + io.prometheus + simpleclient + ${prometheus.version} + + + io.prometheus + simpleclient_common + ${prometheus.version} + + + org.pacesys + openstack4j + ${openstack4j.version} + + + com.oracle + ojdbc6 + ${oracle.version} + + + commons-dbutils + commons-dbutils + ${commons-dbutils.version} + + + com.bocloud + bocloud.vijava + ${vijava.version} + + + log4j + log4j + ${log4j.version} + + + org.dom4j + dom4j + ${dom4j.version} + + + org.snmp4j + snmp4j + ${snmp4j.version} + + + com.huaweicloud.sdk + huaweicloud-sdk-ecs + ${huawei.version} + + + com.huaweicloud.sdk + huaweicloud-sdk-iam + ${huawei.version} + + + com.aliyun + cms20190101 + ${aliyun.cms20190101.version} + + + com.aliyun + tea + ${aliyun.tea.version} + + + + com.baidubce + bce-java-sdk + ${baidubce.version} + + + + software.amazon.awssdk + cloudwatch-metric-publisher + ${awssdk.version} + + + software.amazon.awssdk + ec2 + ${awssdk.version} + + + com.huaweicloud.sdk + huaweicloud-sdk-ecs + ${huawei.version} + + + + com.huaweicloud.sdk + huaweicloud-sdk-ces + ${huawei.version} + + + com.huaweicloud.sdk + huaweicloud-sdk-iam + ${huawei.version} + + + com.huaweicloud.sdk + huaweicloud-sdk-rds + ${huawei.version} + + + com.huaweicloud.sdk + huaweicloud-sdk-as + ${huawei.version} + + + com.huaweicloud.sdk + huaweicloud-sdk-eip + ${huawei.version} + + + com.huaweicloud.sdk + huaweicloud-sdk-evs + ${huawei.version} + + + com.huaweicloud.sdk + huaweicloud-sdk-nat + ${huawei.version} + + + com.huaweicloud.sdk + huaweicloud-sdk-elb + ${huawei.version} + + + org.aspectj + aspectjweaver + + + com.moandjiezana.toml + toml4j + 0.7.2 + + + com.aliyun + asapi + 1.0.6.7-RELEASE + RELEASE + + + com.aliyun + gmsse + 1.1.0 + + + com.ksyun + ksyun-java-sdk + 1.0.6 + + + com.qingcloud + qingcloud-sdk-java + 1.2.2 + + + com.ksyun + ksc-sdk-java-kec + + + com.ksyun + ksc-sdk-java-monitor + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + true + 21 + 21 + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + com.bocloud.mcs.Application + true + false + libs/ + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.6.1 + + + + copy-dependencies + + + jar + jar + runtime + ${project.build.directory}/libs + + + + + + + + + bocloud.booter.tomcat + + 6.5.0-LTS-SZ + + + + org.springframework.boot + spring-boot-starter-web + + + org.hibernate.validator + hibernate-validator + + + + + + + bocloud.booter.tongweb + + 6.5.0-LTS-SZ-TONGWEB + + + + com.tongweb + tongweb-embed + 7.0.E.2 + + + com.tongweb.springboot + tongweb-spring-boot-starter + 2.x.0.RELEASE + + + org.springframework.boot + spring-boot-starter-web + + + org.hibernate.validator + hibernate-validator + + + org.springframework.boot + spring-boot-starter-tomcat + + + + + + + diff --git a/src/main/java/com/bocloud/cms/entity/MonitorAuth.java b/src/main/java/com/bocloud/cms/entity/MonitorAuth.java new file mode 100644 index 0000000..a18fccd --- /dev/null +++ b/src/main/java/com/bocloud/cms/entity/MonitorAuth.java @@ -0,0 +1,23 @@ +package com.bocloud.cms.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * 云平台监控运维认证信息 + * Created by liuyuanyuan on 2019/10/23 17:46 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode +public class MonitorAuth { + private Long id; // ID + private String address; + private String username; + private String password; + private Long vendorId; + +} diff --git a/src/main/java/com/bocloud/ims/entity/CloudVendor.java b/src/main/java/com/bocloud/ims/entity/CloudVendor.java new file mode 100644 index 0000000..fe1d380 --- /dev/null +++ b/src/main/java/com/bocloud/ims/entity/CloudVendor.java @@ -0,0 +1,207 @@ +package com.bocloud.ims.entity; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.megatron.common.utils.DateDeserializer; +import com.megatron.common.utils.DateSerializer; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.springframework.format.annotation.DateTimeFormat; + +import java.math.BigDecimal; +import java.util.Date; + +/** + * 云供应商实体类 + * + * @author wangyu + */ +@Data +@EqualsAndHashCode +public class CloudVendor { + + @JsonSerialize(using = DateSerializer.class) + @JsonDeserialize(using = DateDeserializer.class) + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date gmtCreate; + @JsonSerialize(using = DateSerializer.class) + @JsonDeserialize(using = DateDeserializer.class) + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date gmtModify; + private Long creatorId; + private Long ownerId; + private Long menderId; + private String name; + private String status; + private Boolean deleted; + private String props; + private String remark; + + + private Long id; // ID + private String type; // 平台类型 + private Long envId;// 所在环境 + private String uuid; // butler uuid + private String guid;// 全局唯一标识 + private Boolean isAlarm;// 是否开启告警 + private Boolean isPublic;// 是否为公有云 + private String syncCron;// 云平台同步任务cron + private Long dcId;// 数据中心id + private String version;// 版本 + private String logo;// 平台图标 + private String vendorCategory;// 平台分类 + private String authentication;// 平台认证信息(address,account,password,tenant,access_key,secret_key,domain_name,project_name,region_id,regions,management_ips,subscription_id,client_id,token) + private String region;// 区域 + private String availablitiyZone;// 可用域 + private String location; // 站点位置 + private String supplier; // 供应商 + private String dockerAddress; // 供应商 + + private String remoteSite; // 远程站点 + + private String deployUrl; // 部署机url + private String deployUsername; // 部署机用户名 + private String deployPassword; // 部署机密码 + private Long harborId; + + private String harborName; + + private String regionsName; + + private String envName; + + private String dcName; + + private Long instance; // 实例数量 + + private Long online; // 在线 + + private Long offline; // 离线 + + private Long hostCount;// 主机数量 + + private Long dataStoreCount;// 数据存储数量 + + private Long templateCount;// 模板数量 + + private Long clusterCount;// 集群数量 + + private Long imageCount;// 镜像数量 + + private Long cpuCount;// cpu核数 + + private Long monCount;// 控制节点数量 + + private Long osdCount; // osd节点数量 + + private Long mdsCount;// mds节点数量 + + private Long cpuTotal; // cpu总赫兹 + + private Long cpuUsed; + + private Long cpuUnused; + + private BigDecimal memTotal;// 总内存 + + private BigDecimal memUsed; + + private BigDecimal memUnused; + + private BigDecimal diskTotal; + + private BigDecimal diskUsed; + + private BigDecimal diskUnused; + + private Long tenantCount;// 租户数量 + + private Long networkCount;// 网络数量 + + private Long vsCount;// virtual server数量 + + private Long poolCount;// pool数量 + + private Long nodeCount;// node数量 + + private Long profileCount;// profile数量 + + private Long monitorCount;// monitor数量 + + private Long vpcCount; // vpc数量 + + private Long lparCount;// 逻辑分区数量 + + private Long viosCount; // 虚拟IO服务器数量 + + private Long mdiskCount; // mdisk数量 + + private Long flashCopyCount;// FlashCopy数量 + + private Long fabricCount;// 光纤交换机数量 + + private Long zoneCount; // zone的数量 + + private Long nsCount; // k8s namespace count + + private Long podCount; + + private Long serviceCount; + + private Long deploymentCount; + + private Long aliasCount; // 别名的数量 + + private Long accessPathCount; + + private Long edgeClusterCount;// edge集群 + + private Long groupCount;// NS组 + + private Long transportZoneCount;// 传输区域 + + private Long firewallCount;// 防火墙 + + private Long routerCount;// 路由器 + + private Long switchCount;// 交换机 + + private Long objectUserCount;// 对象用户 + + private Long objectBucketCount;// 存储桶 + + private Long objectRouterCount;// 对象路由 + + + private Long fileUserCount; // xsky 文件用户 + + private Long fileClientCount; // xsky 文件客户端 + + private Long snapshotCount; // 快照 + + private Long fileSystemCount; // 文件系统数量 + + private Long logicPortCount; // 逻辑端口 + + private Long vasCount; // 逻辑vas + + private Long dbCount; // 堡垒机-数据库数量 + + private Long appCount; // 堡垒机-应用数量 + + private Long sessionCount; // 堡垒机-会话数量 + /********** vmware horizon桌面池统计 **************/ + + private Long desktopPools; + + private Long farms; + + private Long sessions; + + private Long users; + + private Long vcenters; + + private Long vendorGroupId; + +} diff --git a/src/main/java/com/bocloud/ims/entity/KvmHost.java b/src/main/java/com/bocloud/ims/entity/KvmHost.java new file mode 100644 index 0000000..86c7a83 --- /dev/null +++ b/src/main/java/com/bocloud/ims/entity/KvmHost.java @@ -0,0 +1,25 @@ +package com.bocloud.ims.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 服务操作业务实现 + * + * @author YuFengYang + * @version 1.0 + * @since 2022/12/30 16:22 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class KvmHost { + + private String uri; + + private Long vendorId; + + private Long id; + +} diff --git a/src/main/java/com/bocloud/ims/enums/CloudProvider.java b/src/main/java/com/bocloud/ims/enums/CloudProvider.java new file mode 100644 index 0000000..971e0cf --- /dev/null +++ b/src/main/java/com/bocloud/ims/enums/CloudProvider.java @@ -0,0 +1,17 @@ +package com.bocloud.ims.enums; + +/** + * 云供应商枚举 + * + * @author wangyu + * @version 1.0 + * @since 2020年1月31日 + * @since 2020年5月9日 modify by aile 增加 XSKY + */ +public enum CloudProvider { + + OPENSTACK, VMWARE, ALIYUN, JDCLOUD, UNKNOWN, CEPH, SCP, CAS, HMC, FUSIONCLOUD, TENCENT, F5, HUAWEI, XEN, POWERVC, + CLOUDOS, SMIS_FABRIC, SMIS_STORAGE, AZURE, KINGCLOUD, KUBERNETES, AWS, QCLOUD, TCE, PQCLOUD, XSKY, MANAGEONE, + JUMPSERVER, HORIZON, TIANYI, HILLSTONE, CISCO, AGILE, NSX,SMARTX,BAIDU,EASYSTACK,HCSO, APSARASTACK,HWACCESS,KVM, + CNWARE,ZSTACK,CLOUDTOWER,CTSTACK; +} diff --git a/src/main/java/com/bocloud/mcs/Application.java b/src/main/java/com/bocloud/mcs/Application.java new file mode 100644 index 0000000..acd3e95 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/Application.java @@ -0,0 +1,21 @@ +package com.bocloud.mcs; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableScheduling; + +/** + * 启动类 + * + * @author zhangyf + */ +@SpringBootApplication +@ComponentScan(value = {"com.bocloud", "com.megatron"}) +@EnableScheduling +public class Application { + + public static void main(String[] args) { + SpringApplication.run(Application.class, args); + } +} diff --git a/src/main/java/com/bocloud/mcs/aspect/AccessAspect.java b/src/main/java/com/bocloud/mcs/aspect/AccessAspect.java new file mode 100644 index 0000000..3963ed1 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/aspect/AccessAspect.java @@ -0,0 +1,53 @@ +package com.bocloud.mcs.aspect; + +import com.megatron.framework.core.CurrentService; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.springframework.context.EnvironmentAware; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; + +/** + * 开关控制中心 + * + * @author lym + * @since 2022/07/18 + */ +@Aspect +@Component +@RequiredArgsConstructor +@Slf4j +public class AccessAspect implements EnvironmentAware { + + private final CurrentService currentService; + + private Environment environment; + + @Override + public void setEnvironment(Environment environment) { + this.environment = environment; + } + + @SneakyThrows + @Around("@annotation(com.bocloud.mcs.aspect.EnableCollector)") + public Object logAround(ProceedingJoinPoint joinPoint) { + String methodName = joinPoint.getSignature().getName(); + EnableCollector annotation = joinPoint.getSignature().getDeclaringType().getMethod(methodName).getAnnotation(EnableCollector.class); + String value = environment.resolvePlaceholders(annotation.value()); + if (Boolean.FALSE.equals(Boolean.valueOf(value))) { + return false; + } + // 是否支持切片,如果不支持,则只允许主节点执行 + if (!annotation.section()) { + if (!currentService.isLeader()) { + log.debug("The current node is not the leader node"); + return false; + } + } + return joinPoint.proceed(); + } +} diff --git a/src/main/java/com/bocloud/mcs/aspect/EnableCollector.java b/src/main/java/com/bocloud/mcs/aspect/EnableCollector.java new file mode 100644 index 0000000..68f0580 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/aspect/EnableCollector.java @@ -0,0 +1,24 @@ +package com.bocloud.mcs.aspect; + +import java.lang.annotation.*; + +/** + * 开启采集控制 + * + * @author lym + * @since 2022/07/18 + */ +@Documented +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface EnableCollector { + + String value() default "true"; + + /** + * 是否支持切片,如果不支持,则只允许主节点执行 + * + * @return 是否支持切片 + */ + boolean section() default false; +} diff --git a/src/main/java/com/bocloud/mcs/collector/BocloudCollector.java b/src/main/java/com/bocloud/mcs/collector/BocloudCollector.java new file mode 100644 index 0000000..948089f --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/BocloudCollector.java @@ -0,0 +1,87 @@ +package com.bocloud.mcs.collector; + +import com.bocloud.mcs.service.internal.CmpCloudVendorService; +import com.megatron.framework.core.CurrentService; +import com.megatron.framework.core.Service; +import com.moandjiezana.toml.Toml; +import io.prometheus.client.Collector; +import lombok.Getter; +import org.apache.commons.lang3.time.DateUtils; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.client.discovery.DiscoveryClient; + +import java.util.*; + +/** + * 性能采集抽象类 + * + * @author zhangyf + */ +public abstract class BocloudCollector extends Collector implements Collector.Describable, InitializingBean { + + protected static final String TIME = "time"; + protected static final String DATA = "data"; + + @Autowired + protected CmpCloudVendorService cmpCloudVendorService; + @Autowired + protected DiscoveryClient discoveryClient; + @Autowired + protected CurrentService currentService; + @Autowired + protected Service service; + @Autowired + @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") + protected Toml collectorConfig; + + /** + * 性能指标缓存 + */ + @Getter + private final Map cache = new HashMap<>(); + + /** + * 默认数据过期时间,10分钟(单位:秒) + */ + protected static final int DEFAULT_EXPIRE = 600; + + /** + * 数据有效时长,单位秒 + */ + @Getter + private int expire = DEFAULT_EXPIRE; + + @SuppressWarnings("unchecked") + @Override + public List collect() { + if (getCache() != null && getCache().size() > 0) { + Date now = new Date(); + Date time = (Date) getCache().get(TIME); + // 数据过期 + if (now.getTime() > DateUtils.addSeconds(time, this.getExpire()).getTime()) { + return new ArrayList<>(); + } else { + return (List) getCache().get(DATA); + } + } else { + return new ArrayList<>(); + } + } + + /** + * 监控指标采集 定时任务 + */ + public abstract void scheduledCollect(); + + @Override + public List describe() { + return new ArrayList<>(); + } + + @Override + public void afterPropertiesSet() { + register(); + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/kvm/ConnectHolder.java b/src/main/java/com/bocloud/mcs/collector/kvm/ConnectHolder.java new file mode 100644 index 0000000..47502bf --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/kvm/ConnectHolder.java @@ -0,0 +1,44 @@ +package com.bocloud.mcs.collector.kvm; + +import lombok.extern.slf4j.Slf4j; +import org.libvirt.Connect; +import org.libvirt.LibvirtException; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +@Slf4j +public class ConnectHolder { + + private static final Map map = new HashMap<>(); + private static Lock lock = new ReentrantLock(); + + private static boolean validate(String url, Connect connect) { + if (connect == null) return false; + try { + if (!connect.isConnected()) return false; + if (!connect.isAlive()) return false; + connect.getHostName(); + return true; + } catch (LibvirtException e) { + log.error("connect {} is not avaliable, message: {}", url, e.getMessage(), e); + return false; + } + } + + public static Connect getConnect(String url) throws LibvirtException { + lock.lock(); + try { + if (!validate(url, map.get(url))) { + log.info("connect {} is not avaliable, recreate", url); + map.put(url, new Connect(url)); + } + log.debug("connect urls: {}", String.join(",", map.keySet())); + return map.get(url); + } finally { + lock.unlock(); + } + } +} diff --git a/src/main/java/com/bocloud/mcs/collector/kvm/KvmHostCollector.java b/src/main/java/com/bocloud/mcs/collector/kvm/KvmHostCollector.java new file mode 100644 index 0000000..b56d6e3 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/kvm/KvmHostCollector.java @@ -0,0 +1,116 @@ +package com.bocloud.mcs.collector.kvm; + +import com.bocloud.mcs.aspect.EnableCollector; +import com.bocloud.mcs.collector.BocloudCollector; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationContext; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * 服务操作业务实现 + * + * @author YuFengYang + * @version 1.0 + * @since 2023/1/12 14:57 + */ + +@Component +@Slf4j +public class KvmHostCollector extends BocloudCollector { + + + @Autowired + private ApplicationContext applicationContext; + + @Value("${collector.kvm.thread.number:20}") + private Integer threadNumber; + + @Override + public List collect() { + return super.collect(); + } + + @Async + @Override + @EnableCollector(value = "${collector.kvm.host.enable:true}") + @Scheduled(initialDelay = 1000L, fixedRate = 60000) + public void scheduledCollect() { +// log.info("collect kvm host metrics start..."); +// GaugeMetricFamily collectTimeUse = new GaugeMetricFamily("kvm_"_HOST + "collect_time_use", "help", +// Arrays.asList("type")); +// ExecutorService executors = Executors.newFixedThreadPool(threadNumber); +// Date now = new Date(); +// List mfs = new ArrayList<>(); +// List list = new ArrayList<>(); +// List vendors = cmpCloudVendorService.list(CloudProvider.KVM); +// List hosts = new ArrayList<>(); +// for (CloudVendor vendor : vendors) { +// List kvmHosts = cmpCloudVendorService.listKvmHost(vendor.getId()); +// hosts.addAll(kvmHosts); +// } +// List labels = Arrays.asList("host_cloud", "host_id"); +// for (KvmHost host : hosts) { +// executors.submit(() -> { +// Connect connect = null; +// try { +// connect = new Connect(host.getUri(), true); +// +//// int vir_node_cpu_stats_user = connect.NodeGetCPUStats(8, "VIR_NODE_CPU_STATS_USER", 0, 0); +//// log.info("---------------------------->{}",vir_node_cpu_stats_user); +//// +// +// NodeInfo nodeInfo = connect.nodeInfo(); +// long freeMemory = connect.getFreeMemory(); +// long memoryTotal = connect.nodeInfo().memory * 1024; +// int cpus = nodeInfo.cpus; +// GaugeMetricFamily memoryFreeData = new GaugeMetricFamily("kvm_"_HOST + "memory_free", "help", +// labels); +// GaugeMetricFamily memoryTotalData = new GaugeMetricFamily("kvm_"_HOST + "memory_total", "help", +// labels); +// GaugeMetricFamily cpuNumData = new GaugeMetricFamily("kvm_"_HOST + "info_virtual_cpus", "help", +// labels); +// +// +// memoryFreeData.addMetric(Arrays.asList(host.getVendorId() + "", host.getId() + ""), freeMemory); +// memoryTotalData.addMetric(Arrays.asList(host.getVendorId() + "", host.getId() + ""), memoryTotal); +// cpuNumData.addMetric(Arrays.asList(host.getVendorId() + "", host.getId() + ""), cpus); +// +// list.add(memoryFreeData); +// list.add(memoryTotalData); +// list.add(cpuNumData); +// } catch (Exception e) { +// log.error("collect node metrics error!" + e.getMessage(), e); +// } finally { +// if (connect != null) { +// try { +// connect.close(); +// } catch (LibvirtException e) { +// log.error("连接关闭异常 {}", e); +// } +// } +// } +// }); +// } +// executors.shutdown(); +// try { +// executors.awaitTermination(5, TimeUnit.MINUTES); +// } catch (InterruptedException e) { +// log.error("kvm not complete within its interval", e); +// Thread.currentThread().interrupt(); +// } +// mfs.addAll(list); +// collectTimeUse.addMetric(Arrays.asList("instance"), (System.currentTimeMillis() - now.getTime())); +// mfs.add(collectTimeUse); +// super.getCache().put(TIME, new Date()); +// super.getCache().put(DATA, mfs); +// log.info("collect kvm vm metrics end! use " + (System.currentTimeMillis() - now.getTime()) / 1000 + "s"); + } + +} + diff --git a/src/main/java/com/bocloud/mcs/collector/kvm/KvmVmCollector.java b/src/main/java/com/bocloud/mcs/collector/kvm/KvmVmCollector.java new file mode 100644 index 0000000..2374cad --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/kvm/KvmVmCollector.java @@ -0,0 +1,112 @@ +package com.bocloud.mcs.collector.kvm; + +import com.bocloud.ims.entity.CloudVendor; +import com.bocloud.ims.entity.KvmHost; +import com.bocloud.ims.enums.CloudProvider; +import com.bocloud.mcs.aspect.EnableCollector; +import com.bocloud.mcs.collector.BocloudCollector; +import com.bocloud.mcs.collector.kvm.vm.KvmVmAdapter; +import io.prometheus.client.GaugeMetricFamily; +import lombok.extern.slf4j.Slf4j; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationContext; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * @author wpj + * @since 2022/6/9 + */ +@Component +@Slf4j +public class KvmVmCollector extends BocloudCollector { + + + @Autowired + private ApplicationContext applicationContext; + + @Value("${collector.kvm.thread.number:20}") + private Integer threadNumber; + + @Override + public List collect() { + return super.collect(); + } + + @Async + @Override + @EnableCollector(value = "${collector.kvm.vm.enable:true}") + @Scheduled(initialDelay = 1000L, fixedRate = 60000) + public void scheduledCollect() { + log.info("collect kvm metrics start..."); + GaugeMetricFamily collectTimeUse = new GaugeMetricFamily("kvm_" + "collect_time_use", "help", + Arrays.asList("type")); + ExecutorService executors = Executors.newFixedThreadPool(threadNumber); + Date now = new Date(); + List mfs = new ArrayList<>(); + List list = new ArrayList<>(); + List vendors = cmpCloudVendorService.list(CloudProvider.KVM); + List hosts = new ArrayList<>(); + for (CloudVendor vendor : vendors) { + List kvmHosts = cmpCloudVendorService.listKvmHost(vendor.getId()); + hosts.addAll(kvmHosts); + } + + for (KvmHost host : hosts) { + executors.submit(() -> { + Connect connect; + try { +// connect = new Connect(host.getUri(), true); + connect = ConnectHolder.getConnect(host.getUri()); + int[] idsOfDomain = connect.listDomains(); + for (int id : idsOfDomain) { + Domain domain = connect.domainLookupByID(id); + if (domain.isActive() != 1) { + log.warn("domain 不处于激活状态 continue"); + continue; + } + Map adapters = applicationContext.getBeansOfType(KvmVmAdapter.class); + for (Map.Entry kvmVmAdapterEntry : adapters.entrySet()) { + kvmVmAdapterEntry.getValue().getGaugeMetricFamilyData(domain, host.getVendorId(), list); + } + } + } catch (Exception e) { + log.error("collect node metrics error!" + e.getMessage(), e); + } +// finally { +// if (connect != null) { +// try { +// connect.close(); +// } catch (LibvirtException e) { +// log.error("连接关闭异常 {}", e); +// } +// } +// } + }); + } + executors.shutdown(); + try { + executors.awaitTermination(5, TimeUnit.MINUTES); + } catch (InterruptedException e) { + log.error("kvm not complete within its interval", e); + Thread.currentThread().interrupt(); + } + mfs.addAll(list); + collectTimeUse.addMetric(Arrays.asList("instance"), (System.currentTimeMillis() - now.getTime())); + mfs.add(collectTimeUse); + super.getCache().put(TIME, new Date()); + super.getCache().put(DATA, mfs); + log.info("collect kvm vm metrics end! use " + (System.currentTimeMillis() - now.getTime()) / 1000 + "s"); + } + +} + diff --git a/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmAdapter.java b/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmAdapter.java new file mode 100644 index 0000000..eb53ed0 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmAdapter.java @@ -0,0 +1,17 @@ +package com.bocloud.mcs.collector.kvm.vm; + +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; + +import java.util.List; + + +public interface KvmVmAdapter { + + List list = null; + + List getGaugeMetricFamilyData(Domain domain, Long vendorId, List result) throws LibvirtException, InterruptedException, NoSuchFieldException, IllegalAccessException; + +} diff --git a/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmCpuMetricData.java b/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmCpuMetricData.java new file mode 100644 index 0000000..c5dadb8 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmCpuMetricData.java @@ -0,0 +1,44 @@ +package com.bocloud.mcs.collector.kvm.vm; + +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; +import lombok.extern.slf4j.Slf4j; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@Slf4j +@Component +public class KvmVmCpuMetricData implements KvmVmAdapter { + + @Override + public List getGaugeMetricFamilyData(Domain domain, Long vendorId, List result) throws LibvirtException, InterruptedException { + List list = new ArrayList<>(); + List labels = Arrays.asList("vmCloud", "vm_uuid", "vmname"); + Double time = domain.getInfo().cpuTime / Math.pow(10, 9); + long c1 = time.longValue(); + int vCpus = domain.getInfo().nrVirtCpu; + log.info("monitoring data vmName=>{}, vmTime=>{},timeLongValue=>{},cpus=>{},systemNow=>{}" + , domain.getName(), time, c1, vCpus, System.currentTimeMillis()); + GaugeMetricFamily cpuTimeData = new GaugeMetricFamily("kvm_" + "domain_info_cpu_time_seconds_total", "help", + labels); + cpuTimeData.addMetric(Arrays.asList(vendorId + "", domain.getUUIDString(), domain.getName()), c1); + list.add(cpuTimeData); + GaugeMetricFamily cpuMaxData = new GaugeMetricFamily("kvm_" + "domain_info_virtual_cpus", "help", + labels); + cpuMaxData.addMetric(Arrays.asList(vendorId + "", domain.getUUIDString(), domain.getName()), vCpus); + + GaugeMetricFamily cpuSystem = new GaugeMetricFamily("kvm_" + "domain_info_cpu_system_time", "help", + labels); + cpuSystem.addMetric(Arrays.asList(vendorId + "", domain.getUUIDString(), domain.getName()), System.currentTimeMillis() / Math.pow(10, 3)); + list.add(cpuMaxData); + list.add(cpuSystem); + result.addAll(list); + return list; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmDiskMetricData.java b/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmDiskMetricData.java new file mode 100644 index 0000000..c6a9372 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmDiskMetricData.java @@ -0,0 +1,65 @@ +package com.bocloud.mcs.collector.kvm.vm; + +import com.alibaba.fastjson.JSONObject; +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.libvirt.Domain; +import org.libvirt.DomainBlockStats; +import org.libvirt.LibvirtException; +import org.springframework.stereotype.Component; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +@Slf4j +@Component +public class KvmVmDiskMetricData implements KvmVmAdapter { + + private String type = "disk"; + private static HashMap typeMap = new HashMap(); + + static { + typeMap.put("rd_req", "读取请求总数"); + typeMap.put("rd_bytes", "读取的数据大小"); + typeMap.put("wr_req", "写入请求总数"); + typeMap.put("wr_bytes", "写入的数据大小"); + typeMap.put("errs", "失败次数"); + } + + + @Override + public List getGaugeMetricFamilyData(Domain domain, Long vendorId, List result) throws LibvirtException, NoSuchFieldException, IllegalAccessException { + List list = new ArrayList<>(); + List labels = Arrays.asList("vmCloud", "vm_uuid", "vmname"); + String xmlDesc = domain.getXMLDesc(0); + Document document = Jsoup.parse(xmlDesc); + // 磁盘 + Elements disks = document.getElementsByTag("devices").get(0).getElementsByTag("disk"); + for (Element disk : disks) { + String dev = disk.getElementsByTag("target").get(0).attr("dev"); + DomainBlockStats domainBlockStats = domain.blockStats(dev); + log.debug("kvm disk {} stats {}", dev, JSONObject.toJSONString(domainBlockStats)); + Class aClass = domainBlockStats.getClass(); + for (String str : typeMap.keySet()) { + Field rxBytes = aClass.getField(str); + long val = (long) rxBytes.get(domainBlockStats); + GaugeMetricFamily nodeData = new GaugeMetricFamily("kvm_" + type + "_" + str, "help", + labels); + nodeData.addMetric(Arrays.asList(vendorId + "", domain.getUUIDString(), domain.getName()), val); + list.add(nodeData); + } + } + result.addAll(list); + return list; + } + + +} diff --git a/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmInterfacesMetricData.java b/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmInterfacesMetricData.java new file mode 100644 index 0000000..73a7ec4 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmInterfacesMetricData.java @@ -0,0 +1,70 @@ +package com.bocloud.mcs.collector.kvm.vm; + +import com.alibaba.fastjson.JSONObject; +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.libvirt.Domain; +import org.libvirt.DomainInterfaceStats; +import org.libvirt.LibvirtException; +import org.springframework.stereotype.Component; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +@Component +@Slf4j +public class KvmVmInterfacesMetricData implements KvmVmAdapter { + + private String type = "interfaces"; + private static HashMap typeMap = new HashMap(); + + static { + typeMap.put("rx_bytes", "接收数据包大小"); + typeMap.put("rx_packets", "接收数据包数量"); + typeMap.put("rx_errs", "接收错误数据包数量"); + typeMap.put("rx_drop", "接收丢弃数据包数量"); + typeMap.put("tx_bytes", "发送数据包大小"); + typeMap.put("tx_packets", "发送数据包数量"); + typeMap.put("tx_errs", "发送错误数据包数量"); + typeMap.put("tx_drop", "发送丢弃数据包数量"); + } + + @Override + public List getGaugeMetricFamilyData(Domain domain, Long vendorId, List result) throws LibvirtException, NoSuchFieldException, IllegalAccessException { + List list = new ArrayList<>(); + List labels = Arrays.asList("vmCloud", "vm_uuid"); + String xmlDesc = domain.getXMLDesc(0); + Document document = Jsoup.parse(xmlDesc); + // 网卡 + Elements interfaces = document.getElementsByTag("devices").get(0).getElementsByTag("interface"); + for (Element inter : interfaces) { + String interName = inter.getElementsByTag("target").get(0).attr("dev"); + DomainInterfaceStats domainInterfaceStats = domain.interfaceStats(interName); + log.debug("kvm interfaces {} stats {}", interName, JSONObject.toJSONString(domainInterfaceStats)); + Class aClass = domainInterfaceStats.getClass(); + for (String str : typeMap.keySet()) { + Field rxBytes = aClass.getField(str); + long val = (long) rxBytes.get(domainInterfaceStats); + GaugeMetricFamily nodeData = new GaugeMetricFamily("kvm_" + type + "_" + str, "help", + labels); + nodeData.addMetric(Arrays.asList(vendorId + "", domain.getUUIDString()), val); + list.add(nodeData); + } + + + } + result.addAll(list); + return list; + } + + +} + diff --git a/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmMemoryMetricData.java b/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmMemoryMetricData.java new file mode 100644 index 0000000..1c64d44 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/kvm/vm/KvmVmMemoryMetricData.java @@ -0,0 +1,51 @@ +package com.bocloud.mcs.collector.kvm.vm; + +import com.alibaba.fastjson.JSONObject; +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; +import lombok.extern.slf4j.Slf4j; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; +import org.libvirt.MemoryStatistic; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +@Component +@Slf4j +public class KvmVmMemoryMetricData implements KvmVmAdapter { + + private String type = "memory_usage"; + + @Override + public List getGaugeMetricFamilyData(Domain domain, Long vendorId, List result) throws LibvirtException { + List list = new ArrayList<>(); + List labels = Arrays.asList("vmCloud", "vm_uuid", "vmname"); + MemoryStatistic[] memoryStatistics = domain.memoryStats(10); + Optional first = Arrays.stream(memoryStatistics).filter(x -> x.getTag() == 4).findFirst(); + double memoryUsage = 0; + try { + MemoryStatistic memoryStatistic = first.get(); + long unusedMemory = memoryStatistic.getValue(); + long maxMemory = domain.getMaxMemory(); + memoryUsage = (maxMemory - unusedMemory) * 100.0 / maxMemory; + log.debug("虚拟机[{}]内存使用率为: {} ", domain.getName(), memoryUsage); + } catch (Exception e) { + log.error("first {}", JSONObject.toJSONString(first)); + log.error("获取内存异常{}", e); + } + + GaugeMetricFamily nodeData = new GaugeMetricFamily("kvm_" + type, "help", + labels); + nodeData.addMetric(Arrays.asList(vendorId + "", domain.getUUIDString(), domain.getName()), (double) Math.round(memoryUsage * 100) / 100); + list.add(nodeData); + result.addAll(list); + return list; + } + + +} + diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/ManageOneEvsDiskCollector.java b/src/main/java/com/bocloud/mcs/collector/manageone/ManageOneEvsDiskCollector.java new file mode 100644 index 0000000..f9d314c --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/ManageOneEvsDiskCollector.java @@ -0,0 +1,105 @@ +package com.bocloud.mcs.collector.manageone; + +import com.alibaba.fastjson.JSONObject; +import com.bocloud.cms.entity.MonitorAuth; +import com.bocloud.ims.entity.CloudVendor; +import com.bocloud.ims.enums.CloudProvider; +import com.bocloud.mcs.aspect.EnableCollector; +import com.bocloud.mcs.collector.BocloudCollector; +import com.bocloud.mcs.collector.manageone.core.ManageOneClient; +import com.bocloud.mcs.service.internal.CmsCloudVendorService; +import com.megatron.common.encrypt.AESEncryptor; +import com.megatron.common.model.GeneralResult; +import com.megatron.common.utils.ListTool; +import io.prometheus.client.GaugeMetricFamily; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * manageone collector + * + * @author zhangyf + * @since 2019-10-12 + */ +@Component +@Slf4j +public class ManageOneEvsDiskCollector extends BocloudCollector implements InitializingBean { + + private final CmsCloudVendorService cmsCloudVendorService; + + List indicatorIds; + JSONObject indicators; + + private String prefix = "bocloud_fusioncloud_evs_disk_"; + + @Value("${collector.manageone.rate.enable:false}") + boolean rateEnable; + @Value("${collector.manageone.rate.value:10}") + double rateValue; + + @Autowired + public ManageOneEvsDiskCollector(CmsCloudVendorService cmsCloudVendorService) { + this.cmsCloudVendorService = cmsCloudVendorService; + } + + @Async + @Override + @EnableCollector(value = "${collector.manageone.evs.disk.enable:false}") + @Scheduled(initialDelay = 3000L, fixedRate = 60000) + public void scheduledCollect() { + Date now = new Date(); + log.info("collect manageone evs disk metrics start..."); + GaugeMetricFamily collectTimeUse = new GaugeMetricFamily(prefix + "collect_time_use", "help", + Collections.singletonList("type")); + List vendors = cmpCloudVendorService.list(CloudProvider.MANAGEONE); + Map map = new ConcurrentHashMap<>(); + for (CloudVendor vendor : vendors) { + try { + ManageOneClient client = new ManageOneClient(rateEnable, rateValue); + try { + // 获取fusioncloud运维侧地址及认证信息 + GeneralResult authResult = cmsCloudVendorService.detailVendorAuth(vendor.getId()); + Assert.isTrue(authResult.isSuccess(), "get vendor operations auth failed , error:" + authResult.getMessage()); + MonitorAuth monitorAuth = JSONObject.parseObject(JSONObject.toJSONString(authResult.getData()), MonitorAuth.class); + Assert.notNull(monitorAuth, "vendor [" + vendor.getName() + "] not found auth config info"); + String password = new AESEncryptor().decrypt(monitorAuth.getPassword(), null); + client.auth(monitorAuth.getAddress(), monitorAuth.getUsername(), password); + } catch (Exception e) { + log.error("auth error ", e); + continue; + } + List objects = client.listCloudVolumes();//云硬盘集合 + if (ListTool.isEmpty(indicatorIds)) { + indicatorIds = client.listIndicatorIds(client.getEvsDiskObjTypeId()); // 支持的监控指标集合IndicatorIds + } + if (indicators == null || indicators.size() == 0) { + indicators = client.listIndicators(indicatorIds); // 指标的数字和描述信息 + } + client.collectMetrics(vendor, objects, indicatorIds, indicators, client.getEvsDiskObjTypeId(), map, prefix); + } catch (RuntimeException e) { + log.error("invoke manageone evs disk get metrics error,", e); + } + } + List mfs = new ArrayList<>(map.values()); + collectTimeUse.addMetric(Collections.singletonList("manageone_evs_disk"), (System.currentTimeMillis() - now.getTime())); + mfs.add(collectTimeUse); + super.getCache().put(TIME, new Date()); + super.getCache().put(DATA, mfs); + log.info("collect manageone evs disk metrics end! use " + (System.currentTimeMillis() - now.getTime()) / 1000 + "s"); + } + + @Override + public void afterPropertiesSet() { + register(); + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/ManageOneHostCollector.java b/src/main/java/com/bocloud/mcs/collector/manageone/ManageOneHostCollector.java new file mode 100644 index 0000000..b57cf3e --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/ManageOneHostCollector.java @@ -0,0 +1,110 @@ +package com.bocloud.mcs.collector.manageone; + +import com.alibaba.fastjson.JSONObject; +import com.bocloud.cms.entity.MonitorAuth; +import com.bocloud.ims.entity.CloudVendor; +import com.bocloud.ims.enums.CloudProvider; +import com.bocloud.mcs.aspect.EnableCollector; +import com.bocloud.mcs.collector.BocloudCollector; +import com.bocloud.mcs.collector.manageone.core.ManageOneClient; +import com.bocloud.mcs.service.internal.CmsCloudVendorService; +import com.megatron.common.encrypt.AESEncryptor; +import com.megatron.common.model.GeneralResult; +import com.megatron.common.utils.ListTool; +import io.prometheus.client.GaugeMetricFamily; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * manageone collector + * + * @author zhangyf + * @since 2019-10-12 + */ +@Component +@Slf4j +public class ManageOneHostCollector extends BocloudCollector implements InitializingBean { + + private final CmsCloudVendorService cmsCloudVendorService; + + List indicatorIds; + JSONObject indicators; + + private String prefix = "bocloud_fusioncloud_host_"; + + @Value("${collector.manageone.rate.enable:false}") + boolean rateEnable; + @Value("${collector.manageone.rate.value:10}") + double rateValue; + + @Autowired + public ManageOneHostCollector(CmsCloudVendorService cmsCloudVendorService) { + this.cmsCloudVendorService = cmsCloudVendorService; + } + + @Async + @Override + @EnableCollector(value = "${collector.manageone.host.enable:true}") + @Scheduled(initialDelay = 1000L, fixedRate = 60000) + public void scheduledCollect() { + Date now = new Date(); + log.info("collect manageone host metrics start..."); + GaugeMetricFamily collectTimeUse = new GaugeMetricFamily(prefix + "collect_time_use", "help", + Collections.singletonList("type")); + List vendors = cmpCloudVendorService.list(CloudProvider.MANAGEONE); + Map map = new ConcurrentHashMap<>(); + for (CloudVendor vendor : vendors) { + try { + Boolean isSyncServer = JSONObject.parseObject(vendor.getAuthentication()).getBoolean("isSyncServer"); + if (isSyncServer == null || !isSyncServer) { + continue; + } + ManageOneClient client = new ManageOneClient(rateEnable, rateValue); + try { + // 获取fusioncloud运维侧地址及认证信息 + GeneralResult authResult = cmsCloudVendorService.detailVendorAuth(vendor.getId()); + Assert.isTrue(authResult.isSuccess(), "get vendor operations auth failed , error:" + authResult.getMessage()); + MonitorAuth monitorAuth = JSONObject.parseObject(JSONObject.toJSONString(authResult.getData()), MonitorAuth.class); + Assert.notNull(monitorAuth, "vendor [" + vendor.getName() + "] not found auth config info"); + String password = new AESEncryptor().decrypt(monitorAuth.getPassword(), null); + client.auth(monitorAuth.getAddress(), monitorAuth.getUsername(), password); + } catch (Exception e) { + log.error("auth error ", e); + continue; + } + List objects = client.listPhysicalHosts();//宿主机集合 + if (ListTool.isEmpty(indicatorIds)) { + indicatorIds = client.listIndicatorIds(client.getHostObjTypeId()); // 支持的监控指标集合IndicatorIds + } + if (indicators == null || indicators.size() == 0) { + indicators = client.listIndicators(indicatorIds); // 指标的数字和描述信息 + log.debug(indicators.toJSONString()); + } + client.collectMetrics(vendor, objects, indicatorIds, indicators, client.getHostObjTypeId(), map, prefix); + } catch (RuntimeException e) { + log.error("invoke manageone host get metrics error,", e); + } + } + List mfs = new ArrayList<>(map.values()); + collectTimeUse.addMetric(Collections.singletonList("manageone_host"), (System.currentTimeMillis() - now.getTime())); + mfs.add(collectTimeUse); + super.getCache().put(TIME, new Date()); + super.getCache().put(DATA, mfs); + log.info("collect manageone host metrics end! use " + (System.currentTimeMillis() - now.getTime()) / 1000 + "s"); + } + + @Override + public void afterPropertiesSet() { + register(); + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/ManageOneVmCollector.java b/src/main/java/com/bocloud/mcs/collector/manageone/ManageOneVmCollector.java new file mode 100644 index 0000000..4a42c19 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/ManageOneVmCollector.java @@ -0,0 +1,110 @@ +package com.bocloud.mcs.collector.manageone; + +import com.alibaba.fastjson.JSONObject; +import com.bocloud.cms.entity.MonitorAuth; +import com.bocloud.ims.entity.CloudVendor; +import com.bocloud.ims.enums.CloudProvider; +import com.bocloud.mcs.aspect.EnableCollector; +import com.bocloud.mcs.collector.BocloudCollector; +import com.bocloud.mcs.collector.manageone.core.ManageOneClient; +import com.bocloud.mcs.service.internal.CmsCloudVendorService; +import com.megatron.common.encrypt.AESEncryptor; +import com.megatron.common.model.GeneralResult; +import com.megatron.common.utils.ListTool; +import io.prometheus.client.GaugeMetricFamily; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * manageone collector + * + * @author zhangyf + * @since 2019-10-12 + */ +@Component +@Slf4j +public class ManageOneVmCollector extends BocloudCollector implements InitializingBean { + + private final CmsCloudVendorService cmsCloudVendorService; + + List indicatorIds; + JSONObject indicators; + + private String prefix = "bocloud_fusioncloud_vm_"; + + @Value("${collector.manageone.rate.enable:false}") + boolean rateEnable; + @Value("${collector.manageone.rate.value:10}") + double rateValue; + + + @Autowired + public ManageOneVmCollector(CmsCloudVendorService cmsCloudVendorService) { + this.cmsCloudVendorService = cmsCloudVendorService; + } + + @Async + @Override + @EnableCollector(value = "${collector.manageone.vm.enable:true}") + @Scheduled(initialDelay = 2000L, fixedRate = 60000) + public void scheduledCollect() { + Date now = new Date(); + log.info("collect manageone vm metrics start..."); + GaugeMetricFamily collectTimeUse = new GaugeMetricFamily(prefix + "collect_time_use", "help", + Collections.singletonList("type")); + List vendors = cmpCloudVendorService.list(CloudProvider.MANAGEONE); + Map map = new ConcurrentHashMap<>(); + for (CloudVendor vendor : vendors) { + try { + String projectId = JSONObject.parseObject(vendor.getAuthentication()).getString("projectId"); + ManageOneClient client = new ManageOneClient(rateEnable, rateValue); + try { + // 获取fusioncloud运维侧地址及认证信息 + GeneralResult authResult = cmsCloudVendorService.detailVendorAuth(vendor.getId()); + Assert.isTrue(authResult.isSuccess(), "get vendor operations auth failed , error:" + authResult.getMessage()); + MonitorAuth monitorAuth = JSONObject.parseObject(JSONObject.toJSONString(authResult.getData()), MonitorAuth.class); + Assert.notNull(monitorAuth, "vendor [" + vendor.getName() + "] not found auth config info"); + String password = new AESEncryptor().decrypt(monitorAuth.getPassword(), null); + client.auth(monitorAuth.getAddress(), monitorAuth.getUsername(), password); + } catch (Exception e) { + log.error("auth error ", e); + continue; + } + if (ListTool.isEmpty(indicatorIds)) { + indicatorIds = client.listIndicatorIds(client.getVmObjTypeId()); // 支持的监控指标集合IndicatorIds + } + if (indicators == null || indicators.size() == 0) { + indicators = client.listIndicators(indicatorIds); // 指标的数字和描述信息 + log.debug(indicators.toJSONString()); + } + List objects = client.listCloudVms(); //虚拟机集合 + objects = objects.stream().filter(object -> object.getString("projectId").equals(projectId)).collect(Collectors.toList()); + client.collectMetrics(vendor, objects, indicatorIds, indicators, client.getVmObjTypeId(), map, prefix); + } catch (RuntimeException e) { + log.error("invoke manageone vm get metrics error,", e); + } + } + List mfs = new ArrayList<>(map.values()); + collectTimeUse.addMetric(Collections.singletonList("fusioncloud_vm"), (System.currentTimeMillis() - now.getTime())); + mfs.add(collectTimeUse); + super.getCache().put(TIME, new Date()); + super.getCache().put(DATA, mfs); + log.info("collect manageone vm metrics end! use " + (System.currentTimeMillis() - now.getTime()) / 1000 + "s"); + } + + @Override + public void afterPropertiesSet() { + register(); + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/bms/BmsCollector.java b/src/main/java/com/bocloud/mcs/collector/manageone/bms/BmsCollector.java new file mode 100644 index 0000000..0bc3c50 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/bms/BmsCollector.java @@ -0,0 +1,138 @@ +package com.bocloud.mcs.collector.manageone.bms; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.bocloud.cms.entity.MonitorAuth; +import com.bocloud.ims.entity.CloudVendor; +import com.bocloud.ims.enums.CloudProvider; +import com.bocloud.mcs.aspect.EnableCollector; +import com.bocloud.mcs.collector.BocloudCollector; +import com.bocloud.mcs.collector.manageone.bms.common.ManageOneBmsClient; +import com.bocloud.mcs.service.internal.CmsCloudVendorService; +import com.megatron.common.encrypt.AESEncryptor; +import com.megatron.common.model.GeneralResult; +import io.prometheus.client.GaugeMetricFamily; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; + +import java.util.*; +import java.util.stream.Collectors; + +@Component +@Slf4j +public class BmsCollector extends BocloudCollector implements InitializingBean { + + private static final String PREFIX = "managerOne_bms_"; + @Autowired + private CmsCloudVendorService cmsCloudVendorService; + + private Integer limit = 5; + + private static final Map METRIC_MAP = new HashMap<>(); + + static { + METRIC_MAP.put("562992903094273","cpu_usage_user"); // 用户空间CPU使用率 + + METRIC_MAP.put("562992903094285","mem_usedPercent"); // 内存使用率 + + METRIC_MAP.put("562992903094292","disk_usedPercent"); // 磁盘使用率 + METRIC_MAP.put("562992903094296","disk_agt_read_bytes_rate"); // 磁盘读速率 + METRIC_MAP.put("562992903094297","disk_agt_read_requests_rate"); // 磁盘读操作速率 + METRIC_MAP.put("562992903094298","disk_agt_write_bytes_rate"); // 磁盘写速率 + METRIC_MAP.put("562992903094299","disk_agt_write_requests_rate"); // 磁盘写操作速率 + METRIC_MAP.put("562992903094301","disk_writeTime"); // 写操作平均耗时 + METRIC_MAP.put("562992903094302","disk_readTime"); // 读操作平均耗时 + + METRIC_MAP.put("562992903094303","net_bitSent"); // 出网带宽 + METRIC_MAP.put("562992903094304","net_bitRecv"); // 入网带宽 + METRIC_MAP.put("562992903094305","net_packetSent"); // 网卡包发送速率 + METRIC_MAP.put("562992903094306","net_packetRecv"); // 网卡包接收速率 + METRIC_MAP.put("562992903094309","net_dropin"); // 接收丢包率 + METRIC_MAP.put("562992903094310","net_dropout"); // 发送丢包率 + } + + /** + * 缓存过期时间,30分钟 + * + * @return + */ + @Override + public int getExpire() { + return 1800000; + } + + @Async + @Override + @EnableCollector(value = "${collector.managerOne.bms.enable:true}") + @Scheduled(initialDelay = 1000L, fixedRate = 120000L) + public void scheduledCollect() { + log.info("BMS => collect managerOne metrics start..."); + Date now = new Date(); + + List mfs = new ArrayList<>(); + GaugeMetricFamily collectTimeUse = new GaugeMetricFamily(PREFIX + "collect_time_use", "help", + Collections.singletonList("type")); + + List vendors = cmpCloudVendorService.list(CloudProvider.MANAGEONE); + + for (CloudVendor vendor : vendors) { + JSONObject auth = JSONObject.parseObject(vendor.getAuthentication()); + String projectId = auth.getString("projectId"); + ManageOneBmsClient manageOneBmsClient = new ManageOneBmsClient(); + GeneralResult authResult = cmsCloudVendorService.detailVendorAuth(vendor.getId()); + Assert.isTrue(authResult.isSuccess(), "get vendor operations auth failed , error:" + authResult.getMessage()); + MonitorAuth monitorAuth = JSONObject.parseObject(JSONObject.toJSONString(authResult.getData()), MonitorAuth.class); + + Assert.notNull(monitorAuth, "vendor [" + vendor.getName() + "] not found auth config info"); + String password = new AESEncryptor().decrypt(monitorAuth.getPassword(), null); + + manageOneBmsClient.setAddr(monitorAuth.getAddress()); + manageOneBmsClient.setUserName(monitorAuth.getUsername()); + manageOneBmsClient.setPassword(password); + manageOneBmsClient.setProjectId(projectId); + Integer offset = 0; + JSONArray jsonArray = manageOneBmsClient.getVms(limit, offset); + while (null != jsonArray && jsonArray.size() != 0){ + // 每五台机器循环获取一次监控数据 + offset += limit; + List vmJson = jsonArray.toJavaList(JSONObject.class); + List vmResId = vmJson.stream().map(jsonObject -> jsonObject.getString("resId")).collect(Collectors.toList()); + JSONObject monitor = manageOneBmsClient.getMonitor(vmResId, METRIC_MAP); + if (null != monitor){ + JSONObject data = monitor.getJSONObject("data"); + if (null != data){ + for (String vmMonitorStr : data.keySet()) { + // 单个云主机监控数据,key为监控指标的key vmMonitorStr为objId + JSONObject vmMonitor = data.getJSONObject(vmMonitorStr); + for (String monitorKey : vmMonitor.keySet()) { + if (METRIC_MAP.containsKey(monitorKey)){ + JSONObject monitorJSONObject = vmMonitor.getJSONObject("monitorKey"); + GaugeMetricFamily cpuMaxData = new GaugeMetricFamily(PREFIX + METRIC_MAP.get(monitorKey), "help", + Arrays.asList("vendorId", "instanceId")); + cpuMaxData.addMetric(Arrays.asList(vendor.getId() + "", vmMonitorStr), monitorJSONObject.getFloatValue("last_collect_data")); + mfs.add(cpuMaxData); + } + } + } + }else { + log.info("本次获取到监控数据为null"); + } + } + jsonArray = manageOneBmsClient.getVms(limit, offset); + } + + } + + + collectTimeUse.addMetric(Collections.singletonList("manager_bms"), (System.currentTimeMillis() - now.getTime())); + mfs.add(collectTimeUse); + super.getCache().put(TIME, new Date()); + super.getCache().put(DATA, mfs); + log.info("collect manager bms metrics end! use " + (System.currentTimeMillis() - now.getTime()) / 1000 + "s"); + } +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/bms/common/ManageCommon.java b/src/main/java/com/bocloud/mcs/collector/manageone/bms/common/ManageCommon.java new file mode 100644 index 0000000..ad3b7cd --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/bms/common/ManageCommon.java @@ -0,0 +1,70 @@ +package com.bocloud.mcs.collector.manageone.bms.common; + +import com.alibaba.fastjson.JSONObject; +import com.megatron.common.enums.PostDataFormat; +import com.megatron.common.http.HttpClient; +import com.megatron.common.model.Result; + +import java.util.Map; + +/** + * @author fanzx + * @create 2021/10/12 11:53 + */ +public class ManageCommon { + + private HttpClient client; + private static final Integer TIMEOUT = 20 * 60 * 1000; + + + /** + * post用户登录功能 用于获取token + * @param url 调用url + * @param header 请求头 固定 + * @param params 请求body -> username and password + * @return + */ + public JSONObject post(String url, Map header, Map params) { + this.client = new HttpClient(TIMEOUT, PostDataFormat.RAW); + try { + client.setDataFormat(PostDataFormat.RAW); + Result result = this.client.post(header, params, url); + client.close(); + String message = result.getMessage(); + return JSONObject.parseObject(message); + } finally { + this.client.close(); + } + } + + /** + * 获取监控数据 + * @param url + * @param header + * @param params + * @return + */ + public JSONObject get(String url, Map header, Map params) { + client = new HttpClient(TIMEOUT, PostDataFormat.RAW); + try { + client.setDataFormat(null); + Result result = client.get(header, params, url); + client.close(); + return JSONObject.parseObject(result.getMessage()); + } finally { + client.close(); + } + } + + public JSONObject put(String url, Map header, Map params) { + client = new HttpClient(TIMEOUT, PostDataFormat.RAW); + try { + Result result = client.put(header, params, url); + client.close(); + return JSONObject.parseObject(result.getMessage()); + } finally { + client.close(); + } + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/bms/common/ManageOneBmsClient.java b/src/main/java/com/bocloud/mcs/collector/manageone/bms/common/ManageOneBmsClient.java new file mode 100644 index 0000000..c2c316a --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/bms/common/ManageOneBmsClient.java @@ -0,0 +1,104 @@ +package com.bocloud.mcs.collector.manageone.bms.common; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.megatron.common.enums.PostDataFormat; +import com.megatron.common.http.HttpClient; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Slf4j +public class ManageOneBmsClient { + private static final String BATCH_GET_METRIC_DATA_URL = "/rest/performance/v1/data-svc/latest-data/action/query"; + private static final String BMS_OBJ_TYPE_ID = "562992903094272"; + private static final String VM_LIST_URL = "/rest/tenant-resource/v1/tenant/resources/CLOUD_BMS"; + private static final String TOKEN_URL = "/rest/plat/smapp/v1/sessions"; + + + private HttpClient client = new HttpClient(PostDataFormat.RAW); + + private String userName; + private String password; + private String addr; + private String projectId; + + + + public String getToken(){ + ManageCommon manageCommon = new ManageCommon(); + Map params = new HashMap<>(); + params.put("grantType", "password"); + params.put("userName", this.userName); + params.put("value" ,this.password); + log.info("BMS => 获取token接口参数为{}", JSONObject.toJSONString(params)); + JSONObject jsonObject = manageCommon.put(this.buildUrl(TOKEN_URL), buildHeader(null), params); + log.info("BMS => 获取token接口返回: {}", jsonObject.toJSONString()); + log.info("BMS => token为: {}", jsonObject.getString("accessSession")); + return jsonObject.getString("accessSession"); + } + + private String buildUrl(String tokenUrl) { + return this.addr + tokenUrl; + } + + private Map buildHeader(String token) { + HashMap header = new HashMap<>(); + header.put("Content-Type", "application/json;charset=UTF-8"); + header.put("Accept", "application/json"); + if (null != token){ + header.put("x-auth-token", token); + } + return header; + } + + + public JSONArray getVms(Integer limit, Integer offset) { + Map header = this.buildHeader(this.getToken()); + ManageCommon manageCommon = new ManageCommon(); + HashMap params = new HashMap<>(); + params.put("limit", limit); + params.put("offset", offset); + log.info("BMS => 获取bms列表参数为:{}, url为:{}, 请求头为{}", JSONObject.toJSONString(params), buildUrl(VM_LIST_URL), + JSONObject.toJSONString(header)); + try { + JSONObject jsonObject = manageCommon.get(buildUrl(VM_LIST_URL), header, params); + log.info("BMS => 获取到bms列表为{}", jsonObject.toJSONString()); + return jsonObject.getJSONArray("objList"); + }catch (Exception e){ + log.error("获取bms 云主机列表异常", e); + return null; + } + } + + public JSONObject getMonitor(List vmIds, Map METRIC_MAP) { + Map header = this.buildHeader(this.getToken()); + ManageCommon manageCommon = new ManageCommon(); + HashMap params = new HashMap<>(); + params.put("obj_type_id", BMS_OBJ_TYPE_ID); + params.put("indicator_ids", JSONObject.parseArray(JSONObject.toJSONString(new ArrayList<>(METRIC_MAP.keySet())))); + params.put("obj_ids", JSONObject.parseArray(JSONObject.toJSONString(vmIds))); + log.info("BMS => 获取bms监控参数为{}", JSONObject.toJSONString(params)); + try { + JSONObject jsonObject = manageCommon.post(buildUrl(BATCH_GET_METRIC_DATA_URL), header, params); + log.info("BMS => 获取到bms监控数据为 {}", jsonObject.toJSONString()); + return jsonObject; + }catch (Exception e){ + log.error("BMS => 获取bms 云主机列表异常", e); + return null; + } + } + + + + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/CceCollector.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/CceCollector.java new file mode 100644 index 0000000..6807221 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/CceCollector.java @@ -0,0 +1,142 @@ +package com.bocloud.mcs.collector.manageone.cce; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.bocloud.cms.entity.MonitorAuth; +import com.bocloud.ims.entity.CloudVendor; +import com.bocloud.ims.enums.CloudProvider; +import com.bocloud.mcs.aspect.EnableCollector; +import com.bocloud.mcs.collector.BocloudCollector; +import com.bocloud.mcs.collector.manageone.cce.common.ContainerCluster; +import com.bocloud.mcs.collector.manageone.cce.common.ManageOneCceClient; +import com.bocloud.mcs.collector.manageone.cce.common.ManagerMonitorEntity; +import com.bocloud.mcs.common.Authentication; +import com.bocloud.mcs.service.internal.CmsCloudVendorService; +import com.bocloud.mcs.service.internal.ImsCloudServerService; +import com.megatron.common.encrypt.AESEncryptor; +import com.megatron.common.model.GeneralResult; +import io.prometheus.client.GaugeMetricFamily; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; + +import java.util.*; +import java.util.stream.Collectors; + +@Component +@Slf4j +public class CceCollector extends BocloudCollector implements InitializingBean { + + private static final String PREFIX = "managerOne_cce_"; + @Autowired + private CmsCloudVendorService cmsCloudVendorService; + @Autowired + private ImsCloudServerService imsCloudServerService; + + private Integer limit = 5; + + private static final Map METRIC_MAP = new HashMap<>(); + + static { +// METRIC_MAP.put("磁盘写入速率","diskWriteRate"); // 用户空间CPU使用率 +// METRIC_MAP.put("磁盘读取速率","diskReadRate"); // 内存使用率 +// METRIC_MAP.put("CPU使用率","cpuUsage"); // 磁盘使用率 +// METRIC_MAP.put("物理内存使用率","memUsage"); // 磁盘读速率 +// METRIC_MAP.put("接收Bps","recvBytesRate"); // 磁盘读操作速率 +// METRIC_MAP.put("接收Pps","recvPackRate"); // 磁盘写速率 +// METRIC_MAP.put("显存使用率","gpuMemUsage"); // 磁盘写操作速率 +// METRIC_MAP.put("GPU使用率","gpuUtil"); // 写操作平均耗时 +// METRIC_MAP.put("发送Bps","sendBytesRate"); // 写操作平均耗时 +// METRIC_MAP.put("发送Pps","sendPackRate"); // 写操作平均耗时 + + + + METRIC_MAP.put("CPU使用率","cpuUsage"); + METRIC_MAP.put("内存使用率","memUsedRate"); + + METRIC_MAP.put("磁盘使用率","diskUsedRate"); + METRIC_MAP.put("网卡接收速率","recvBytesRate"); // Bytes/Second + METRIC_MAP.put("网卡发送速率","sendBytesRate"); // Bytes/Second + } + + /** + * 缓存过期时间,30分钟 + * + * @return + */ + @Override + public int getExpire() { + return 1800000; + } + + @Async + @Override + @EnableCollector(value = "${collector.managerOne.cce.enable:true}") + @Scheduled(initialDelay = 1000L, fixedRate = 120000L) + public void scheduledCollect() { + log.info("CCE => collect managerOne metrics start..."); + Date now = new Date(); + + List mfs = new ArrayList<>(); + GaugeMetricFamily collectTimeUse = new GaugeMetricFamily(PREFIX + "collect_time_use", "help", + Collections.singletonList("type")); + + List vendors = cmpCloudVendorService.list(CloudProvider.MANAGEONE); + + for (CloudVendor vendor : vendors) { + try { + Authentication authentication = JSON.parseObject(vendor.getAuthentication(), Authentication.class); + GeneralResult authResult = cmsCloudVendorService.detailVendorAuth(vendor.getId()); + Assert.isTrue(authResult.isSuccess(), "get vendor operations auth failed , error:" + authResult.getMessage()); + MonitorAuth monitorAuth = JSONObject.parseObject(JSONObject.toJSONString(authResult.getData()), MonitorAuth.class); + Assert.notNull(monitorAuth, "vendor [" + vendor.getName() + "] not found auth config info"); + String addr = monitorAuth.getAddress().replace("oc", "aom"); + log.info("aom地址为: {}", addr); + String password = new AESEncryptor().decrypt(authentication.getPassword(), "BocloudCMP"); + ManageOneCceClient manageOneCceClient = new ManageOneCceClient(authentication.getAddress(), addr,authentication.getAccount(), password, authentication.getDomainName(), authentication.getProjectId()); + + List cceContainer = imsCloudServerService.getCceContainer(vendor.getId()); + List clusterUuids = cceContainer.stream().map(ContainerCluster::getClusterUuid).collect(Collectors.toList()); + log.info("CCE => 获取到cce列表为{}", JSONObject.toJSONString(cceContainer)); + for (String value : METRIC_MAP.values()) { + for (String clusterUuid : clusterUuids) { + JSONObject monitorResource = manageOneCceClient.getMonitor(Arrays.asList(clusterUuid), value); + if (monitorResource!= null) { + ManagerMonitorEntity managerMonitorEntity = JSONObject.parseObject(monitorResource.toJSONString(), ManagerMonitorEntity.class); + for (ManagerMonitorEntity.Metrics metric : managerMonitorEntity.getMetrics()) { + + List dimensions = metric.getMetric().getDimensions(); + List dataPoints = metric.getDataPoints(); + + if (dimensions!= null && dimensions.size() > 0) { + for (int i = 0; i < dimensions.size(); i++) { + if (clusterUuid.contains(dimensions.get(i).getValue())){ + String cceUuid = dimensions.get(i).getValue(); + String val = dataPoints.get(i).getStatistics().get(0).getValue(); + GaugeMetricFamily cpuMaxData = new GaugeMetricFamily(PREFIX + metric.getMetric().getMetricName(), "help", + Arrays.asList("vendorId", "instanceId")); + cpuMaxData.addMetric(Arrays.asList(vendor.getId() + "", cceUuid), Double.parseDouble(val)); + mfs.add(cpuMaxData); + } + } + } + } + } + } + } + }catch (Exception e) { + log.error("collect managerOne metrics error", e); + } + } + + collectTimeUse.addMetric(Collections.singletonList("manager_cce"), (System.currentTimeMillis() - now.getTime())); + mfs.add(collectTimeUse); + super.getCache().put(TIME, new Date()); + super.getCache().put(DATA, mfs); + log.info("collect manager bms metrics end! use " + (System.currentTimeMillis() - now.getTime()) / 1000 + "s"); + } +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ContainerCluster.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ContainerCluster.java new file mode 100644 index 0000000..8007ba6 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ContainerCluster.java @@ -0,0 +1,160 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.megatron.common.encrypt.AESEncryptor; +import com.megatron.common.encrypt.Encryptor; +import com.megatron.common.model.RequestContext; +import com.megatron.common.utils.DateDeserializer; +import com.megatron.common.utils.DateSerializer; +import com.megatron.common.utils.DaySerializer; +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import org.springframework.format.annotation.DateTimeFormat; + +import java.util.Date; + +/** + * @ClassName: ContainerCluster + * @Description: 容器Kubernetes 集群 + * @Author lxy + * @Date 2022/7/14 17:54 + */ +@Slf4j +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ContainerCluster { + + private Long id;// ID + private String regionId;// 所属域 + + private Long vendorId;// 所属云平台 + private String clusterUuid;// 集群ID + private String clusterType;// 集群类型 + // 集群创建时间 + private Date created; + // 集群版本 + private String version; + // 集群Docker版本。 + private String dockerVersion; + // 集群API Server访问地址,包含内网访问地址以及公网访问地址。 + private String masterUrl; + // 集群安全组ID。 + private String securityGroupUuid; + /** + * 集群运行状态,取值: initial:集群创建中。 failed:集群创建失败。 running:集群运行中。 updating:集群升级中。 updating_failed:集群升级失败。 scaling:集群伸缩中。 + * stopped:集群已经停止运行。 deleting:集群删除中。 deleted:集群已经被删除。 delete_failed:集群删除失败。 + */ + // @Column("status") + // private String status; + + /** + * Pod网络地址段,必须是有效的私有网段,即以下网段及其子网: + *

+ * 10.0.0.0/8 172.16-31.0.0/12-16 192.168.0.0/16 不能与VPC及VPC内已有Kubernetes集群使用的网段重复,创建成功后不能修改。 + */ + private String subnetCidr; + // 集群专有网络ID,创建集群时必传。 + private String vpcUuid; + private String vpcName; + // 虚拟交换机ID,创建集群时可选择1~3个虚拟交换机。为保证集群高可用,建议选择不同可用区的虚拟交换机。 + private String vswitchUuid; + // 集群所属可用区ID。 + private String zoneId; + // 托管版集群类型,面向托管集群: ack.pro.small:专业托管集群。 ack.standard:标准托管集群。 + private String clusterSpec; + // 集群删除保护 + private Boolean deletionProtection; + private String eipUuid; + private String loadBalancerUuid; + private String resourceGroupUuid;// 资源组id + // SSH登录密码。密码规则为8~30 个字符,且至少同时包含三项(大小写字母、数字和特殊符号),和key_pair二选一。 + public String loginPassword; + // 密钥对名称,和login_password二选一。 + public String keyPair; + + // 网络插件 + private String capabilitiesNetwork; + // kube-proxy代理模式 + private String capabilitiesProxyModel; + + // HSC START + // 子网uuid + private String subnetUuid; + private String subnetName; + // 容器网段 + private String containerCidr; + // 服务网段 + private String serviceCidr; + // HCS END + + // 自服务 start + private Long tenantId;// 租户ID + private String tenantName; // 租户名称 + + private Long projectId;// 项目ID + private String projectName; // 项目名称 + + private String uuid;// uuid,对应子订单号 + + @JsonSerialize(using = DaySerializer.class) + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date expiredTime;// 过期时间 + // 所属申请类别 + private RequestContext.Catalog catalog; + // 所属业务id + private Long businessId; + // 所属业务名称 + private String businessName; + + private Boolean isRecycle = false; + @JsonSerialize(using = DaySerializer.class) + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date recycleTime;// 回收时间 + // 自服务 end + + private String regionName; + private String vpcSafetyType;//安全类型 + private String vpcFunctionType; + private String applicationName; //应用系统 + private String resourceSetName; //资源集 + private String imageId;//自定义镜像 + + + @Getter(AccessLevel.NONE) + @Setter(AccessLevel.NONE) + private Encryptor encryptor = new AESEncryptor(); + + + @JsonSerialize( + using = DateSerializer.class + ) + @JsonDeserialize( + using = DateDeserializer.class + ) + @DateTimeFormat( + pattern = "yyyy-MM-dd HH:mm:ss" + ) + private Date gmtCreate; + + @JsonSerialize( + using = DateSerializer.class + ) + @JsonDeserialize( + using = DateDeserializer.class + ) + @DateTimeFormat( + pattern = "yyyy-MM-dd HH:mm:ss" + ) + private Date gmtModify; + private Long creatorId; + private Long ownerId; + private Long menderId; + private String name; + private String status; + private Boolean deleted; + private String props; + private String remark; + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneAuth.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneAuth.java new file mode 100644 index 0000000..4220939 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneAuth.java @@ -0,0 +1,63 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +/** + * 华为manage one认证实体类 + * + * @author wangyu + * @version 1.0 + * @since 2018年9月16日 + * + */ +public class ManageOneAuth { + + private ManageOneIdentity identity; + private ManageOneScope scope; + + /** + * + */ + public ManageOneAuth() { + super(); + } + + /** + * @param identity + * @param scope + */ + public ManageOneAuth(ManageOneIdentity identity, ManageOneScope scope) { + super(); + this.identity = identity; + this.scope = scope; + } + + /** + * @return the identity + */ + public ManageOneIdentity getIdentity() { + return identity; + } + + /** + * @param identity + * the identity to set + */ + public void setIdentity(ManageOneIdentity identity) { + this.identity = identity; + } + + /** + * @return the scope + */ + public ManageOneScope getScope() { + return scope; + } + + /** + * @param scope + * the scope to set + */ + public void setScope(ManageOneScope scope) { + this.scope = scope; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneCatalog.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneCatalog.java new file mode 100644 index 0000000..ff78580 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneCatalog.java @@ -0,0 +1,91 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +import java.util.List; + +public class ManageOneCatalog { + + private String id; + private String name; + private String type; + private List endpoints; + + /** + * + */ + public ManageOneCatalog() { + super(); + } + + /** + * @param name + * @param type + * @param endpoints + */ + public ManageOneCatalog(String name, String type, List endpoints) { + super(); + this.name = name; + this.type = type; + this.endpoints = endpoints; + } + + /** + * @return the id + */ + public String getId() { + return id; + } + + /** + * @param id + * the id to set + */ + public void setId(String id) { + this.id = id; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name + * the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the type + */ + public String getType() { + return type; + } + + /** + * @param type + * the type to set + */ + public void setType(String type) { + this.type = type; + } + + /** + * @return the endpoints + */ + public List getEndpoints() { + return endpoints; + } + + /** + * @param endpoints + * the endpoints to set + */ + public void setEndpoints(List endpoints) { + this.endpoints = endpoints; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneCceClient.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneCceClient.java new file mode 100644 index 0000000..8d922a8 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneCceClient.java @@ -0,0 +1,187 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.megatron.common.enums.PostDataFormat; +import com.megatron.common.http.CommonResultBuilder; +import com.megatron.common.http.HttpClient; +import com.megatron.common.model.Result; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.Header; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; + +import java.util.*; + +/** + * @ Description: @ Author :丁明威 @ Date :Created in 17:07 2018/11/14 @ Modified + * By: + */ +@Data +@Slf4j +@NoArgsConstructor +public class ManageOneCceClient { + + private static final String GET_METRIC_DATA_URL = "/v1/{project_id}/ams/metricdata"; + + private String username;// 管理员账号 + private String password;// 管理员密码 + private String domain;// 所属域 + private String project;// 项目ID + private boolean accessable; + private ManageOneToken token;// token信息 + private String error; // 错误信息 + private HttpClient client = new HttpClient(60 * 1000, PostDataFormat.RAW); + private String monitorUrl; + private String monitorUserName; + private String monitorPwd; + private String monitorToken; + private String monitorRoaRand; + private String tokenUrl; + + + public ManageOneCceClient(String tokenUrl,String monitorUrl, String userName, String pwd, String domainName, String projectId) { + this.tokenUrl = tokenUrl; + this.monitorUrl = monitorUrl; + this.username = userName; + this.password = pwd; + this.domain = domainName; + this.project = projectId; + initToken(); + } + + private void initToken() { + init(); + } + + + /** + * 初始化连接 + */ + private void init() { + String url = this.tokenUrl + "/v3/auth/tokens"; + ManageOneDomain domain = new ManageOneDomain(this.domain); + ManageOneUser user = new ManageOneUser(this.username, this.password, domain); + ManageOnePassword password = new ManageOnePassword(user); + List methods = new ArrayList(); + methods.add("password"); + ManageOneIdentity identity = new ManageOneIdentity(password, methods); + + ManageOneProject project = new ManageOneProject(this.project); + ManageOneScope scope = new ManageOneScope(project); + ManageOneAuth auth = new ManageOneAuth(identity, scope); + Map params = new HashMap(); + params.put("auth", auth); + HttpPost httpPost = new HttpPost(url); + httpPost.addHeader("accept", "application/json"); + httpPost.addHeader("content-type", "application/json"); + log.info("CCE => 参数{}", JSONObject.toJSONString(params)); + StringEntity httpEntity = new StringEntity(JSONObject.toJSON(params).toString(), "UTF-8"); + httpPost.setEntity(httpEntity); + try { + CloseableHttpResponse httpResponse = client.getHttpClient().execute(httpPost); + log.info("CCE => 获取token地址{},参数{}, 返回{}", url, httpEntity, JSONObject.toJSONString(httpResponse)); + Result result = new CommonResultBuilder().build(httpResponse.getStatusLine().getStatusCode()); + if (result.isFailed()) { + this.accessable = false; + log.error("CCE => Driver init error : {}", result.getMessage()); + this.error = result.getMessage(); + return; + } + JSONObject content = JSONObject + .parseObject(IOUtils.toString(httpResponse.getEntity().getContent(), "utf-8")); + this.token = JSONObject.parseObject(content.getString("token"), ManageOneToken.class); + if (null == this.token) { + this.accessable = false; + log.error("CCE => Driver init error : {}", "Token is empty !"); + this.error = "Token is empty !"; + return; + } + String authToken = findToken(httpResponse); + if (StringUtils.isEmpty(authToken)) { + this.accessable = false; + log.error("CCE => Driver init error : {}", "Token is empty !"); + this.error = "CCE => Token is empty !"; + return; + } + token.setToken(authToken); + this.accessable = true; + log.info("CCE => Driver init token : {}", JSON.toJSONString(token)); + } catch (Exception e) { + log.error("CCE => Get error message:", e); + this.accessable = false; + } + } + + /** + * 从响应头部获取会话X-Subject-Token + * + * @param httpResponse + * @return + */ + private String findToken(CloseableHttpResponse httpResponse) { + String token = ""; + Header[] headers = httpResponse.getHeaders("X-Subject-Token"); + if (null == headers || headers.length == 0) { + return token; + } + for (Header header : headers) { + if ("X-Subject-Token".equalsIgnoreCase(header.getName())) { + token = header.getValue(); + break; + } + } + return token; + } + + + + + private Map buildHeader() { + Map header = new HashMap(); + header.put("accept", "application/json"); + header.put("content-type", "application/json"); + header.put("X-Auth-Token", this.token.getToken()); + return header; + } + + public JSONObject getMonitor(List instanceIds, String metricName) { + ManageOneCommon manageOneCommon = new ManageOneCommon(); + String dimensionName = "clusterId";// 维度名称。长度最短为1,最大为32。 + String timerange = "-1.-1.5"; // 表示查询近5分钟数据 + long period = 60; //周期 + JSONObject metricObj = new JSONObject(); + metricObj.put("namespace", "PAAS.AGGR"); + metricObj.put("metricName", metricName); + ArrayList dimensions = new ArrayList<>(); + instanceIds.forEach(instanceid -> { + JSONObject dimension = new JSONObject(); + dimension.put("name", dimensionName); + dimension.put("value", instanceid); + dimensions.add(dimension); + }); + metricObj.put("dimensions", dimensions); + Map params = new HashMap<>(); + params.put("metrics", Arrays.asList(metricObj)); + params.put("period", period); + params.put("timerange", timerange); + params.put("statistics", Collections.singletonList("average")); + String url = buildUrl(GET_METRIC_DATA_URL, this.project); + log.info("CCE => 获取cce监控参数为{}, 路径为: {}, 参数为{}", JSONObject.toJSONString(params), url, JSONObject.toJSONString(params)); + Map header = this.buildHeader(); + JSONObject jsonObject = manageOneCommon.post(url, header, params); + log.info("CCE => 获取到cce监控数据为 {}", jsonObject.toJSONString()); + return jsonObject; + } + + private String buildUrl(String url, String projectId) { + return (this.monitorUrl + url).replace("{project_id}", projectId); + } + + +} \ No newline at end of file diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneCommon.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneCommon.java new file mode 100644 index 0000000..f7de37a --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneCommon.java @@ -0,0 +1,70 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +import com.alibaba.fastjson.JSONObject; +import com.megatron.common.enums.PostDataFormat; +import com.megatron.common.http.HttpClient; +import com.megatron.common.model.Result; + +import java.util.Map; + +/** + * @author fanzx + * @create 2021/10/12 11:53 + */ +public class ManageOneCommon { + + private HttpClient client; + private static final Integer TIMEOUT = 20 * 60 * 1000; + + + /** + * post用户登录功能 用于获取token + * @param url 调用url + * @param header 请求头 固定 + * @param params 请求body -> username and password + * @return + */ + public JSONObject post(String url, Map header, Map params) { + this.client = new HttpClient(TIMEOUT, PostDataFormat.RAW); + try { + client.setDataFormat(PostDataFormat.RAW); + Result result = this.client.post(header, params, url); + client.close(); + String message = result.getMessage(); + return JSONObject.parseObject(message); + } finally { + this.client.close(); + } + } + + /** + * 获取监控数据 + * @param url + * @param header + * @param params + * @return + */ + public JSONObject get(String url, Map header, Map params) { + client = new HttpClient(TIMEOUT, PostDataFormat.RAW); + try { + client.setDataFormat(null); + Result result = client.get(header, params, url); + client.close(); + return JSONObject.parseObject(result.getMessage()); + } finally { + client.close(); + } + } + + public JSONObject put(String url, Map header, Map params) { + client = new HttpClient(TIMEOUT, PostDataFormat.RAW); + try { + Result result = client.put(header, params, url); + client.close(); + return JSONObject.parseObject(result.getMessage()); + } finally { + client.close(); + } + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneDomain.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneDomain.java new file mode 100644 index 0000000..c779319 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneDomain.java @@ -0,0 +1,62 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +public class ManageOneDomain { + + private String id; + private String name; + + /** + * + */ + public ManageOneDomain() { + super(); + } + + /** + * @param id + * @param name + */ + public ManageOneDomain(String id, String name) { + super(); + this.id = id; + this.name = name; + } + + /** + * @param name + */ + public ManageOneDomain(String name) { + super(); + this.name = name; + } + + /** + * @return the id + */ + public String getId() { + return id; + } + + /** + * @param id + * the id to set + */ + public void setId(String id) { + this.id = id; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name + * the name to set + */ + public void setName(String name) { + this.name = name; + } +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneEndpoint.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneEndpoint.java new file mode 100644 index 0000000..d6a4672 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneEndpoint.java @@ -0,0 +1,89 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +public class ManageOneEndpoint { + + private String id; + private String regionId; + private String region; + private String url; + + /** + * + */ + public ManageOneEndpoint() { + super(); + } + + /** + * @param regionId + * @param region + * @param url + */ + public ManageOneEndpoint(String regionId, String region, String url) { + super(); + this.regionId = regionId; + this.region = region; + this.url = url; + } + + /** + * @return the id + */ + public String getId() { + return id; + } + + /** + * @param id + * the id to set + */ + public void setId(String id) { + this.id = id; + } + + /** + * @return the regionId + */ + public String getRegionId() { + return regionId; + } + + /** + * @param regionId + * the regionId to set + */ + public void setRegionId(String regionId) { + this.regionId = regionId; + } + + /** + * @return the region + */ + public String getRegion() { + return region; + } + + /** + * @param region + * the region to set + */ + public void setRegion(String region) { + this.region = region; + } + + /** + * @return the url + */ + public String getUrl() { + return url; + } + + /** + * @param url + * the url to set + */ + public void setUrl(String url) { + this.url = url; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneIdentity.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneIdentity.java new file mode 100644 index 0000000..28ee68d --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneIdentity.java @@ -0,0 +1,57 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +import java.util.List; + +public class ManageOneIdentity { + + private ManageOnePassword password; + private List methods; + + /** + * + */ + public ManageOneIdentity() { + super(); + } + + /** + * @param password + * @param methods + */ + public ManageOneIdentity(ManageOnePassword password, List methods) { + super(); + this.password = password; + this.methods = methods; + } + + /** + * @return the password + */ + public ManageOnePassword getPassword() { + return password; + } + + /** + * @param password + * the password to set + */ + public void setPassword(ManageOnePassword password) { + this.password = password; + } + + /** + * @return the methods + */ + public List getMethods() { + return methods; + } + + /** + * @param methods + * the methods to set + */ + public void setMethods(List methods) { + this.methods = methods; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOnePassword.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOnePassword.java new file mode 100644 index 0000000..fda302a --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOnePassword.java @@ -0,0 +1,37 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +public class ManageOnePassword { + + private ManageOneUser user; + + /** + * + */ + public ManageOnePassword() { + super(); + } + + /** + * @param user + */ + public ManageOnePassword(ManageOneUser user) { + super(); + this.user = user; + } + + /** + * @return the user + */ + public ManageOneUser getUser() { + return user; + } + + /** + * @param user + * the user to set + */ + public void setUser(ManageOneUser user) { + this.user = user; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneProject.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneProject.java new file mode 100644 index 0000000..33e0802 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneProject.java @@ -0,0 +1,87 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +public class ManageOneProject { + + private String id; + private String name; + private ManageOneDomain domain; + + /** + * + */ + public ManageOneProject() { + super(); + } + + /** + * @param id + */ + public ManageOneProject(String id) { + super(); + this.id = id; + } + + /** + * @param domain + */ + public ManageOneProject(ManageOneDomain domain) { + super(); + this.domain = domain; + } + + /** + * @param name + * @param domain + */ + public ManageOneProject(String name, ManageOneDomain domain) { + super(); + this.name = name; + this.domain = domain; + } + + /** + * @return the id + */ + public String getId() { + return id; + } + + /** + * @param id + * the id to set + */ + public void setId(String id) { + this.id = id; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name + * the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the domain + */ + public ManageOneDomain getDomain() { + return domain; + } + + /** + * @param domain + * the domain to set + */ + public void setDomain(ManageOneDomain domain) { + this.domain = domain; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneResponse.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneResponse.java new file mode 100644 index 0000000..6422bf7 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneResponse.java @@ -0,0 +1,21 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; + +@Data +public class ManageOneResponse { + + @JSONField(name = "job_id") + public String jobId; + + private ErrorInfo error; + + @Data + public static class ErrorInfo{ + + public String message; + + private String code; + } +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneRole.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneRole.java new file mode 100644 index 0000000..c03322f --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneRole.java @@ -0,0 +1,53 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +public class ManageOneRole { + + private String id; + private String name; + + /** + * + */ + public ManageOneRole() { + super(); + } + + /** + * @param name + */ + public ManageOneRole(String name) { + super(); + this.name = name; + } + + /** + * @return the id + */ + public String getId() { + return id; + } + + /** + * @param id + * the id to set + */ + public void setId(String id) { + this.id = id; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name + * the name to set + */ + public void setName(String name) { + this.name = name; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneScope.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneScope.java new file mode 100644 index 0000000..7156f97 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneScope.java @@ -0,0 +1,37 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +public class ManageOneScope { + + private ManageOneProject project; + + /** + * + */ + public ManageOneScope() { + super(); + } + + /** + * @param project + */ + public ManageOneScope(ManageOneProject project) { + super(); + this.project = project; + } + + /** + * @return the project + */ + public ManageOneProject getProject() { + return project; + } + + /** + * @param project + * the project to set + */ + public void setProject(ManageOneProject project) { + this.project = project; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneToken.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneToken.java new file mode 100644 index 0000000..760c051 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneToken.java @@ -0,0 +1,181 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +import com.alibaba.fastjson.annotation.JSONField; + +import java.util.Date; +import java.util.List; + +/** + * 华为manage one会话Token + * + * @author wangyu + * @version 1.0 + * @since 2018年9月16日 + * + */ +public class ManageOneToken { + + private String token; + @JSONField(format = "yyyy-MM-dd'T'HH:mm:ss.SSS") + private Date expiresAt; + private List methods; + private List catalog; + private List roles; + private ManageOneProject project; + @JSONField(format = "yyyy-MM-dd'T'HH:mm:ss.SSS") + private Date issuedAt; + private ManageOneUser user; + + /** + * + */ + public ManageOneToken() { + super(); + } + + /** + * @param token + * @param expiresAt + * @param methods + * @param catalog + * @param roles + * @param project + * @param issuedAt + * @param user + */ + public ManageOneToken(String token, Date expiresAt, List methods, List catalog, + List roles, ManageOneProject project, Date issuedAt, ManageOneUser user) { + super(); + this.token = token; + this.expiresAt = expiresAt; + this.methods = methods; + this.catalog = catalog; + this.roles = roles; + this.project = project; + this.issuedAt = issuedAt; + this.user = user; + } + + /** + * @return the token + */ + public String getToken() { + return token; + } + + /** + * @param token + * the token to set + */ + public void setToken(String token) { + this.token = token; + } + + /** + * @return the expiresAt + */ + @JSONField(format = "yyyy-MM-dd HH:mm:ss") + public Date getExpiresAt() { + return expiresAt; + } + + /** + * @param expiresAt + * the expiresAt to set + */ + public void setExpiresAt(Date expiresAt) { + this.expiresAt = expiresAt; + } + + /** + * @return the methods + */ + public List getMethods() { + return methods; + } + + /** + * @param methods + * the methods to set + */ + public void setMethods(List methods) { + this.methods = methods; + } + + /** + * @return the catalog + */ + public List getCatalog() { + return catalog; + } + + /** + * @param catalog + * the catalog to set + */ + public void setCatalog(List catalog) { + this.catalog = catalog; + } + + /** + * @return the roles + */ + public List getRoles() { + return roles; + } + + /** + * @param roles + * the roles to set + */ + public void setRoles(List roles) { + this.roles = roles; + } + + /** + * @return the project + */ + public ManageOneProject getProject() { + return project; + } + + /** + * @param project + * the project to set + */ + public void setProject(ManageOneProject project) { + this.project = project; + } + + /** + * @return the issuedAt + */ + @JSONField(format = "yyyy-MM-dd HH:mm:ss") + public Date getIssuedAt() { + return issuedAt; + } + + /** + * @param issuedAt + * the issuedAt to set + */ + public void setIssuedAt(Date issuedAt) { + this.issuedAt = issuedAt; + } + + /** + * @return the user + */ + public ManageOneUser getUser() { + return user; + } + + /** + * @param user + * the user to set + */ + public void setUser(ManageOneUser user) { + this.user = user; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneUser.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneUser.java new file mode 100644 index 0000000..65176ba --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManageOneUser.java @@ -0,0 +1,89 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + +public class ManageOneUser { + + private String id; + private String name; + private String password; + private ManageOneDomain domain; + + /** + * + */ + public ManageOneUser() { + super(); + } + + /** + * @param name + * @param password + * @param domain + */ + public ManageOneUser(String name, String password, ManageOneDomain domain) { + super(); + this.name = name; + this.password = password; + this.domain = domain; + } + + /** + * @return the id + */ + public String getId() { + return id; + } + + /** + * @param id + * the id to set + */ + public void setId(String id) { + this.id = id; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name + * the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the password + */ + public String getPassword() { + return password; + } + + /** + * @param password + * the password to set + */ + public void setPassword(String password) { + this.password = password; + } + + /** + * @return the domain + */ + public ManageOneDomain getDomain() { + return domain; + } + + /** + * @param domain + * the domain to set + */ + public void setDomain(ManageOneDomain domain) { + this.domain = domain; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManagerMonitorEntity.java b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManagerMonitorEntity.java new file mode 100644 index 0000000..0e24333 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/cce/common/ManagerMonitorEntity.java @@ -0,0 +1,89 @@ +package com.bocloud.mcs.collector.manageone.cce.common; + + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ManagerMonitorEntity { + + /** + * { + * "errorCode":"SVCSTG_AMS_2000000", + * "errorMessage":"success", + * "metrics":[ + * { + * "metric":{ + * "namespace":"abc", + * "metricName":"def", + * "dimensions":[ + * { + * "name":"ghi", + * "value":"lmn" + * } + * ] + * }, + * "dataPoints":[ + * { + * "timestamp":"1467892800000", + * "unit":"Percent", + * "statistics":[ + * { + * "statistic":"maximum", + * "value":"23" + * } + * ] + * } + * ] + * } + * ] + * } + */ + + private String errorCode; + private String errorMessage; + private List metrics; + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class Metrics{ + private Metric metric; + private List dataPoints; + } + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class Metric{ + private String namespace; + private String metricName; + private List dimensions; + } + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class Dimensions{ + private String name; + private String value; + } + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class DataPoints{ + private String timestamp; + private String unit; + private List statistics; + } + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class Statistics{ + private String statistic; + private String value; + } +} diff --git a/src/main/java/com/bocloud/mcs/collector/manageone/core/ManageOneClient.java b/src/main/java/com/bocloud/mcs/collector/manageone/core/ManageOneClient.java new file mode 100644 index 0000000..c61d80f --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/manageone/core/ManageOneClient.java @@ -0,0 +1,294 @@ +package com.bocloud.mcs.collector.manageone.core; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.bocloud.ims.entity.CloudVendor; +import com.google.common.util.concurrent.RateLimiter; +import com.megatron.common.enums.PostDataFormat; +import com.megatron.common.http.HttpClient; +import com.megatron.common.model.Result; +import io.prometheus.client.GaugeMetricFamily; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.Assert; + +import java.text.DecimalFormat; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.*; +import java.util.stream.Collectors; + +/** + * ManageOne + * + * @author zhangyf + * @since 2019-10-17 + */ +@Slf4j +public class ManageOneClient { + + private final boolean rateEnable; + + public static RateLimiter rateLimiter; + + private final HttpClient client = new HttpClient(PostDataFormat.RAW); + private String endpoint; + private String token; + /** + * roaRand + */ + private String roaRand; + + public ManageOneClient(boolean rateEnable, double rateValue) { + this.rateEnable = rateEnable; + if (rateLimiter == null) { + rateLimiter = RateLimiter.create(rateValue); + } + } + + /** + * 运维侧权限认证 + * + * @param endpoint 端点 + * @param user 用户名 + * @param password 密码 + */ + public void auth(String endpoint, String user, String password) { + this.endpoint = endpoint; + String url = endpoint + "/rest/plat/smapp/v1/oauth/token"; + Map params = new HashMap<>(4); + params.put("grantType", "password"); + params.put("userName", user); + params.put("value", password); + + Map header = new HashMap<>(3); + header.put("accept", "application/json"); + header.put("content-type", "application/json"); + Result result = client.put(header, params, url); + Assert.isTrue(result.isSuccess(), result.getMessage()); + JSONObject content = JSONObject.parseObject(result.getMessage()); + this.token = content.getString("accessSession"); + this.roaRand = content.getString("roaRand"); + } + + public void collectMetrics(CloudVendor vendor, List objects, List indicatorIds, JSONObject indicators, + Long objTypeId, Map map, String prefix) { + List objIds = Optional.ofNullable(objects).orElse(new ArrayList<>()).stream().map(vm -> vm.getString("resId")).collect(Collectors.toList()); + Map objIdNativeIdMap = new HashMap<>(); //resId,nativeId(openstack里资源的id) + Optional.ofNullable(objects).orElse(new ArrayList<>()).forEach(object -> objIdNativeIdMap.put(object.getString("resId"), object)); + + JSONObject metrics = this.metrics(objIds, objTypeId, indicatorIds); + objIds.forEach(objId -> { + JSONObject metric = metrics.getJSONObject(objId); + Optional.ofNullable(metric).orElse(new JSONObject()).forEach((key, value) -> { + JSONObject indicator = indicators.getJSONObject(key); + if (!Optional.ofNullable(map.get(key)).isPresent()) { + GaugeMetricFamily gaugeMetricFamily = new GaugeMetricFamily(prefix + indicator.getString("indicator_name"), + indicator.getString("zh_cn"), Arrays.asList("vendorId", "id", "name", "instanceUuid")); + map.put(key, gaugeMetricFamily); + } + GaugeMetricFamily gaugeMetricFamily = map.get(key); + assert metric != null; + double datapointValue = 0D; + JSONArray series = metric.getJSONObject(key).getJSONArray("series"); + if (series != null && !series.isEmpty()) { + series.sort(Comparator.comparingInt(o -> JSON.parseObject(o.toString()).size())); + List list = series.stream().filter(o -> !JSON.parseObject(o.toString()).isEmpty()).collect(Collectors.toList()); + if (!list.isEmpty()) { + List values = new ArrayList<>(JSON.parseObject(String.valueOf(list.get(list.size() - 1))).values()); + if (!values.isEmpty() && values.get(0) != null) { + DecimalFormat df = new DecimalFormat("#.00"); + datapointValue = Double.parseDouble(df.format(Double.parseDouble(String.valueOf(values.get(0))))); + } + } + } + gaugeMetricFamily.addMetric(Arrays.asList(String.valueOf(vendor.getId()), objIdNativeIdMap.get(objId).getString("nativeId"), + objIdNativeIdMap.get(objId).getString("name"), objIdNativeIdMap.get(objId).getString("nativeId")), datapointValue); + }); + }); + } + + /** + * 获取宿主机列表 + * + * @return 宿主机列表 + */ + public List listPhysicalHosts() { + int pageNo = 1; + int pageSize = 1000; + List list = null; + while (true) { + String url = this.endpoint + "/rest/cmdb/v1/instances/SYS_PhysicalHost"; + Map params = new HashMap<>(); + params.put("pageSize", pageSize); + params.put("pageNo", pageNo++); + params.put("contentSelector", JSONObject.toJSONString(Arrays.asList("resId", "nativeId", "name"))); + Result result = client.get(defaultHeader(), params, url); + JSONObject content = JSONObject.parseObject(result.getMessage()); + Assert.isTrue(result.isSuccess(), result.getMessage()); + if (list == null) { + list = new ArrayList<>(content.getIntValue("totalNum")); + } + list.addAll(JSONObject.parseArray(content.getString("objList"), JSONObject.class)); + if (content.getIntValue("totalPageNo") == content.getIntValue("currentPage")) { + break; + } + } + return list; + } + + /** + * 获取虚拟机列表 + * + * @return 虚拟机列表 + */ + public List listCloudVms() { + return listTenantResource("CLOUD_VM"); + } + + + /** + * 获取云硬盘列表 + * + * @return 云硬盘列表 + */ + public List listCloudVolumes() { + return listTenantResource("CLOUD_VOLUME"); + } + + /** + * 租户资源查询 + * + * @param className 资源类型,和《租户资源北向模型》中的资源类型与视图类型保持一致。 + * @return + */ + private List listTenantResource(String className) { + int limit = 1000; + int offset = 0; + List list = null; + while (true) { + String url = this.endpoint + "/rest/tenant-resource/v1/tenant/resources/" + className; + JSONObject params = new JSONObject(); + params.put("limit", limit); + params.put("offset", offset); + Result result = client.get(defaultHeader(), params, url); + JSONObject content = JSONObject.parseObject(result.getMessage()); + Assert.isTrue(result.isSuccess(), result.getMessage()); + if (list == null) { + list = new ArrayList<>(content.getIntValue("totalNum")); + } + list.addAll(JSONObject.parseArray(content.getString("objList"), JSONObject.class)); + offset = limit * content.getIntValue("currentPage"); + if (content.getIntValue("totalPageNo") == content.getIntValue("currentPage")) { + break; + } + } + return list; + } + + /** + * 获取最新性能数据 + * + * @param objIds 资源resId集合 + * @param objTypeId 资源对象类型Id + * @param indicatorIds 监控指标集合IndicatorIds + * @return 最新性能数据 + */ + public JSONObject metrics(List objIds, Long objTypeId, List indicatorIds) { + JSONObject jsonObject = new JSONObject(); + String url = this.endpoint + "/rest/performance/v1/data-svc/history-data/action/query"; + int size = objIds.size(); + long step = 100 / indicatorIds.size(); + int i = 0; + LocalDateTime now = LocalDateTime.now(); + long endTime = now.toInstant(ZoneOffset.of("+8")).toEpochMilli(); + long beginTime = now.minusMinutes(10).toInstant(ZoneOffset.of("+8")).toEpochMilli(); + while (size > 0) { + size -= step; + List filterObjIds = objIds.stream().skip(step * i).limit(step).collect(Collectors.toList()); + i++; + JSONObject params = new JSONObject(); + params.put("obj_type_id", objTypeId); + params.put("indicator_ids", indicatorIds); + params.put("obj_ids", filterObjIds); + params.put("interval", "ORIGINAL"); + params.put("range", "BEGIN_END_TIME"); + params.put("begin_time", beginTime); + params.put("end_time", endTime); + if (rateEnable) { + ManageOneClient.rateLimiter.acquire(1); + } + Result result = client.post(defaultHeader(), params, url); + Assert.isTrue(result.isSuccess(), result.getMessage()); + jsonObject.putAll(JSONObject.parseObject(result.getMessage()).getJSONObject("data")); + } + return jsonObject; + } + + /** + * 查询指标的数字和描述信息 + * + * @return 指标的数字和描述信息 + */ + public JSONObject listIndicators(List indicatorIds) { + String url = this.endpoint + "/rest/performance/v1/mgr-svc/indicators"; + Result result = client.post(defaultHeader(), indicatorIds, url); + Assert.isTrue(result.isSuccess(), result.getMessage()); + return JSONObject.parseObject(result.getMessage()).getJSONObject("data"); + } + + /** + * 获取objType支持的监控指标集合IndicatorIds + * + * @param objTypeId objTypeId + * @return indicatorIds + */ + public List listIndicatorIds(Long objTypeId) { + String url = this.endpoint + "/rest/performance/v1/mgr-svc/obj-types/" + objTypeId + "/indicators"; + Result result = client.get(defaultHeader(), null, url); + Assert.isTrue(result.isSuccess(), result.getMessage()); + return JSONObject.parseObject(result.getMessage()).getJSONObject("data").getJSONArray("indicator_ids").toJavaList(Long.class); + } + + /** + * 默认header,包含认证信息 + * + * @return header + */ + private Map defaultHeader() { + Map header = new HashMap<>(); + header.put("Accept", "application/json"); + header.put("Content-Type", "application/json;charset=UTF-8"); + header.put("X-Auth-Token", this.token); + header.put("roaRand", this.roaRand); + return header; + } + + /** + * 获取虚拟机对象类型对应的id(objTypeId) + * + * @return 虚拟机类型id + */ + public Long getVmObjTypeId() { + return 562958543355904L; + } + + /** + * 获取宿主机对象类型对应的id(objTypeId) + * + * @return 宿主机类型id + */ + public Long getHostObjTypeId() { + return 1407379178520576L; + } + + /** + * 获取云硬盘对象类型对应的id(objTypeId) + * + * @return 云硬盘类型id + */ + public Long getEvsDiskObjTypeId() { + return 562967133290496L; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/tianyicloud/TianyiCloudCollector.java b/src/main/java/com/bocloud/mcs/collector/tianyicloud/TianyiCloudCollector.java new file mode 100644 index 0000000..56e164d --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/tianyicloud/TianyiCloudCollector.java @@ -0,0 +1,113 @@ +package com.bocloud.mcs.collector.tianyicloud; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.bocloud.ims.entity.CloudVendor; +import com.bocloud.ims.enums.CloudProvider; +import com.bocloud.mcs.aspect.EnableCollector; +import com.bocloud.mcs.collector.BocloudCollector; +import com.bocloud.mcs.collector.tianyicloud.core.TianyiProvider; +import com.bocloud.mcs.common.Authentication; +import com.megatron.common.encrypt.DESEncryptor; +import io.prometheus.client.GaugeMetricFamily; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.*; + +@Component +@Slf4j +public class TianyiCloudCollector extends BocloudCollector implements InitializingBean { + + private static final String PREFIX = "tianyi_vm_"; + + @Value("${metrics.tianyi.batchSize:100}") + private Integer batchSize = 100; + + /** + * 缓存过期时间,30分钟 + * + * @return + */ + @Override + public int getExpire() { + return 1800000; + } + + @Async + @Override + @EnableCollector(value = "${collector.tianyi.enable:true}") + @Scheduled(initialDelay = 1000L, fixedRate = 120000L) + public void scheduledCollect() { + Date now = new Date(); + log.info("collect tianyi vm metrics start..."); + GaugeMetricFamily collectTimeUse = new GaugeMetricFamily(PREFIX + "collect_time_use", "help", + Collections.singletonList("type")); + List mfs = new ArrayList<>(); + List labels = Arrays.asList("vendor_id", "instanceId", "region", "instanceUuid"); + List vendors = cmpCloudVendorService.list(CloudProvider.CTSTACK); + for (CloudVendor vendor : vendors) { + if (StringUtils.isBlank(vendor.getAuthentication())) { + continue; + } + Authentication authentication = JSONArray.parseObject(vendor.getAuthentication(), Authentication.class); + String accessKey = new DESEncryptor().decrypt(authentication.getAccessKey().trim(), "BocloudCMP").trim(); + String secretKey = new DESEncryptor().decrypt(authentication.getSecretKey().trim(), "BocloudCMP").trim(); + String projectId = authentication.getProjectId(); + for (String region : authentication.getRegions()) { + try { + TianyiProvider tianyiProvider = new TianyiProvider(accessKey, secretKey, projectId, region, authentication.getUrl()); + List instanceIds = tianyiProvider.instanceList(); + + if (instanceIds == null || instanceIds.isEmpty()) { + log.warn("获取到云主机列表为null, 云平台名称: {}, 可用区:{}", vendor.getName(), region); + continue; + } + // 分割instanceIds,控制每批获取监控的云主机数量 + for (List instanceIdsBatch : splitList(instanceIds, batchSize)) { + List metric = tianyiProvider.getMetric(instanceIdsBatch); + for (JSONObject jsonObject : metric) { // 每个metric为一个vm的监控数据 + List itemList = JSONObject.parseArray(jsonObject.getJSONArray("itemList").toJSONString(), JSONObject.class); + for (JSONObject metricItem : itemList) { + //windows机器,磁盘监控数据返回值会包括.C: ,暂定把点后面的删除 + String name = (PREFIX + metricItem.getString("itemName")); + if (name.contains(".")) { + name = name.substring(0, name.indexOf(".")); + } + GaugeMetricFamily gmf = new GaugeMetricFamily(name.replace("./", ""), "help", labels); + gmf.addMetric(Arrays.asList(vendor.getId() + "", jsonObject.getString("deviceUUID"), jsonObject.getString("regionID"), jsonObject.getString("deviceUUID")), + Double.parseDouble(metricItem.getString("value"))); + mfs.add(gmf); + } + } + } + } catch (Exception e) { + log.error("获取云主机监控信息失败, 云平台名称: {}, 可用区:{}, 异常原因", vendor.getName(), region, e); + } + } + } + collectTimeUse.addMetric(Arrays.asList("tianyi"), (System.currentTimeMillis() - now.getTime())); + mfs.add(collectTimeUse); + super.getCache().put(TIME, new Date()); + super.getCache().put(DATA, mfs); + log.info("collect tianyi metrics end! use " + (System.currentTimeMillis() - now.getTime()) / 1000 + "s"); + } + + public static List> splitList(List originalList, int batchSize) { + List> batches = new ArrayList<>(); + + for (int i = 0; i < originalList.size(); i += batchSize) { + int end = Math.min(i + batchSize, originalList.size()); + List batch = new ArrayList<>(originalList.subList(i, end)); + batches.add(batch); + } + + return batches; + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/tianyicloud/core/MessageDigestUtils.java b/src/main/java/com/bocloud/mcs/collector/tianyicloud/core/MessageDigestUtils.java new file mode 100644 index 0000000..a043abc --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/tianyicloud/core/MessageDigestUtils.java @@ -0,0 +1,147 @@ +package com.bocloud.mcs.collector.tianyicloud.core; + + +import lombok.extern.slf4j.Slf4j; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * 加密工具类 + * + *
+ * 采用消息摘要加密算法,支持如下算法:
+ * MD2     The MD2 message digest algorithm as defined in RFC 1319.
+ * MD5     The MD5 message digest algorithm as defined in RFC 1321.
+ * SHA-1   Hash algorithms defined in the FIPS PUB 180-2.
+ * SHA-256 SHA-256 is a 256-bit hash function intended to provide 128 bits of security against collision attacks,
+ * SHA-384 while SHA-512 is a 512-bit hash function intended to provide 256 bits of security. A 384-bit hash may
+ * SHA-512 be obtained by truncating the SHA-512 output.
+ * 
+ * + * @author Shawpin Shi + * @since 2016-9-1 + */ +@Slf4j +public class MessageDigestUtils { + + private final static char[] HEX_ARRAY = "0123456789abcdef".toCharArray(); + + /** + * MD5加密(128bit,16byte,32hex)
+ * Message Digest Algorithm MD5(消息摘要算法第五版) + */ + public static final String MD5 = "MD5"; + + /** + * SHA-1加密(160bit,20byte,40hex)
+ * 安全哈希算法(Secure Hash Algorithm)
+ * SHA家族的五个算法,分别是SHA-1、SHA-224、SHA-256、SHA-384,和SHA-512, 由美国国家安全局(NSA)所设计,
+ * 并由美国国家标准与技术研究院(NIST)发布; 后四者有时并称为SHA-2。
+ * 特别注意:Java目前不支持SHA-224算法。 + */ + public static final String SHA_1 = "SHA-1"; + + /** + * SHA-256加密(256bit,64hex) + */ + public static final String SHA_256 = "SHA-256"; + + /** + * SHA-384加密(384bit,96hex) + */ + public static final String SHA_384 = "SHA-384"; + + /** + * SHA-512加密(512bit,64byte,128hex) + */ + public static final String SHA_512 = "SHA-512"; + + /** + * 根据指定的算法名称加密 + * + * @author Shawpin Shi + * @since 2016-9-1 + */ + public static String encrypt(String source, String algorithmName) { + return encrypt(source, algorithmName, StandardCharsets.UTF_8); + } + + /** + * 根据指定的算法名称加密(支持指定字符串序列化字符集) + * + * @author Shawpin Shi + * @since 2020-12-03 + */ + public static String encrypt(String source, String algorithmName, Charset charset) { + String ciphertext = null; + MessageDigest md; + try { + md = MessageDigest.getInstance(algorithmName); + byte[] digest = md.digest(source.getBytes(charset)); + ciphertext = byteToHexString(digest); + } catch (NoSuchAlgorithmException e) { + log.error(e.getMessage(), e); + } + + return ciphertext; + } + + /** + * 根据指定的算法和盐值加密 + * + * @author Shawpin Shi + * @since 2016-9-1 + */ + public static String encryptWithSalt(String source, String salt, String algorithmName) { + String newSource = source + "@" + salt; + return encrypt(newSource, algorithmName); + } + + /** + * 将字节数转化成16进制的字符串 + * + * @author Shawpin Shi + * @since 2016-9-1 + * @since 2017-9-29 调用高效的工具类 + */ + private static String byteToHexString(byte[] bytes) { + int len = bytes.length; + if (len > Integer.MAX_VALUE >> 1) { + throw new RuntimeException("字节数组的长度不能超过" + (Integer.MAX_VALUE >> 1)); + } + + int newLen = len << 1;// len * 2 + char[] hexChars = new char[newLen]; + + int v, index; + + for (int i = 0; i < len; i++) { + v = bytes[i] & 0xFF; // 保留低8位,高24位置0 + index = i << 1;// i * 2 + hexChars[index] = HEX_ARRAY[v >>> 4]; + hexChars[index + 1] = HEX_ARRAY[v & 0x0F]; + } + return new String(hexChars); + } + + + /** + * Test + */ + public static void main(String[] args) { + String source = "0aece0ff48aafb9c481826d13ef5e3c8b74e44b4584b28a0fe499aee108227e8db8470f4b9f9ac5d2ab508a9ff84312da9f1646a8b5ed6570fd2f27790460e47"; + System.out.println("Source String:" + source); + System.out.println("Encrypted String:"); + System.out.println("Use MD5: " + MessageDigestUtils.encrypt(source, MessageDigestUtils.MD5)); + System.out.println("Use SHA-1: " + MessageDigestUtils.encrypt(source, MessageDigestUtils.SHA_1)); + System.out.println("Use SHA-256: " + MessageDigestUtils.encrypt(source, MessageDigestUtils.SHA_256)); + System.out.println("Use SHA-384: " + MessageDigestUtils.encrypt(source, MessageDigestUtils.SHA_384)); + System.out.println("Use SHA-512: " + MessageDigestUtils.encrypt(source, MessageDigestUtils.SHA_512)); + + System.out.println("MD5 Salt: " + MessageDigestUtils.encryptWithSalt(source, "sxp", MessageDigestUtils.MD5)); + } + +} diff --git a/src/main/java/com/bocloud/mcs/collector/tianyicloud/core/TianyiHeaderUtils.java b/src/main/java/com/bocloud/mcs/collector/tianyicloud/core/TianyiHeaderUtils.java new file mode 100644 index 0000000..cc5160e --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/tianyicloud/core/TianyiHeaderUtils.java @@ -0,0 +1,126 @@ +package com.bocloud.mcs.collector.tianyicloud.core; + +import com.alibaba.fastjson.JSONObject; +import com.megatron.common.enums.HttpRequestMethod; +import com.megatron.common.utils.MapTools; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.time.DateFormatUtils; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class TianyiHeaderUtils { + + public static Map createHeaders(HttpRequestMethod method, Map headers, Map params, + String accessKey, String secretKey, String ctUserId) { + Map newHeaders = new HashMap<>(); + Map headersOfAuth = new HashMap<>(); + try { + if (!MapTools.isEmpty(headers)) { + newHeaders.putAll(headers); + } + String hybridDate = DateFormatUtils.format(new Date(), "yyyyMMdd'T'HHmmss'Z'"); + headersOfAuth.put("hybrid-date", hybridDate); + String requestId = UUID.randomUUID().toString().replace("-", ""); + headersOfAuth.put("ctyun-hybrid-request-id", requestId); + String forSignature = forSignature(method, headersOfAuth, params); + byte[] kdate = kdate(hybridDate, accessKey, secretKey); + byte[] signatureBytes = hmacSHA256(forSignature.getBytes(StandardCharsets.UTF_8), kdate); + String signature = new String(Base64.encodeBase64(signatureBytes), StandardCharsets.UTF_8); + List keyList = new ArrayList<>(headersOfAuth.keySet()); + keyList = keyList.stream().sorted(Comparator.comparing(String::toString)).collect(Collectors.toList()); + String headersValue = String.join(";", keyList); + String authorization = accessKey + " Header=" + headersValue + " Signature=" + signature; + + newHeaders.put("Content-Type", "application/json"); + newHeaders.put("hybrid-date", hybridDate); + newHeaders.put("ctyun-hybrid-request-id", requestId); + newHeaders.put("Hybrid-Authorization", authorization); + newHeaders.put("ctUserId", ctUserId); + + } catch (Exception e) { + log.error("构造请求头出现异常", e); + } + return newHeaders; + + } + + private static String forSignature(HttpRequestMethod method, Map headersOfAuth, Map params) { + List keyList = new ArrayList<>(headersOfAuth.keySet()); + keyList = keyList.stream().sorted(Comparator.comparing(String::toString)).collect(Collectors.toList()); + StringBuilder headerBuffer = new StringBuilder(); + for (int i = 0; i < keyList.size(); i++) { + String key = keyList.get(i); + headerBuffer.append(key).append(':').append(headersOfAuth.get(key)); + if (i != keyList.size() - 1) { + headerBuffer.append('\n'); + } + } + String campmocalHeader = headerBuffer.toString(); + + String query = null; + String body = null; + String encodedValue; + Object value; + if (method == HttpRequestMethod.GET) { + if (params != null && !params.isEmpty()) { + keyList = new ArrayList<>(params.keySet()); + keyList = keyList.stream().sorted(Comparator.comparing(String::toString)).collect(Collectors.toList()); + // aa=1&bb=2 + StringBuilder queryBuffer = new StringBuilder(); + for (String key : keyList) { + if (!queryBuffer.isEmpty()) { + queryBuffer.append('&'); + } + value = params.get(key); + if (value == null) { + value = ""; + } + encodedValue = URLEncoder.encode(String.valueOf(value), StandardCharsets.UTF_8); + queryBuffer.append(key).append('=').append(encodedValue); + } + query = queryBuffer.toString(); + } + } else if (method == HttpRequestMethod.POST) { + if (params != null && !params.isEmpty()) { + body = JSONObject.toJSONString(params); + body = MessageDigestUtils.encrypt(body, MessageDigestUtils.SHA_256); + } + } + + String forSignature = campmocalHeader; + if (null != query) { + forSignature = forSignature + "\n" + query; + } + if (null != body) { + forSignature = forSignature + "\n" + body; + } + return forSignature; + } + + private static byte[] kdate(String hybridDate, String accessKey, String secretKey) throws Exception { + byte[] ktime = hmacSHA256(hybridDate.getBytes(StandardCharsets.UTF_8), secretKey.getBytes(StandardCharsets.UTF_8)); + byte[] kAk = hmacSHA256(accessKey.getBytes(StandardCharsets.UTF_8), ktime); + byte[] kdate = hmacSHA256(hybridDate.substring(0, 8).getBytes(StandardCharsets.UTF_8), kAk); + return kdate; + } + + /** + * HmacSHA256算法,返回的结果始终是32位 + * + * @param contentBytes 待加密的内容 + * @param secretBytes 密钥 + * @return 加密后的内容 + */ + private static byte[] hmacSHA256(byte[] contentBytes, byte[] secretBytes) throws Exception { + Mac hmacSha256 = Mac.getInstance("HmacSHA256"); + hmacSha256.init(new SecretKeySpec(secretBytes, "HmacSHA256")); + return hmacSha256.doFinal(contentBytes); + } +} \ No newline at end of file diff --git a/src/main/java/com/bocloud/mcs/collector/tianyicloud/core/TianyiProvider.java b/src/main/java/com/bocloud/mcs/collector/tianyicloud/core/TianyiProvider.java new file mode 100644 index 0000000..3f99ce4 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/collector/tianyicloud/core/TianyiProvider.java @@ -0,0 +1,206 @@ +package com.bocloud.mcs.collector.tianyicloud.core; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.megatron.common.enums.HttpRequestMethod; +import com.megatron.common.enums.PostDataFormat; +import com.megatron.common.http.CommonResultBuilder; +import com.megatron.common.http.HttpClient; +import com.megatron.common.http.ResultBuilder; +import com.megatron.common.model.Result; +import com.megatron.common.utils.ListTool; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.Assert; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +@Data +public class TianyiProvider { + private String accessKey; + private String secretKey; + private String projectId; + private String ctUserId; + private String regionId; + public static final Integer SUCCESS_STATUS_CODE = 800; + public String PROTOCOL; + public Integer PAGE_SIZE = 50; + private final HttpClient client = new HttpClient(60 * 1000, PostDataFormat.RAW); + + protected ResultBuilder resultBuilder = new CommonResultBuilder(); + + public TianyiProvider(String ak, String sk, String projectId, String reginId, String url) { + this.accessKey = ak; + this.secretKey = sk; + this.ctUserId = projectId; + this.regionId = reginId; + this.PROTOCOL = url; + } + + private Map buildHeader(HttpRequestMethod method, Map headers, Map params) { + return TianyiHeaderUtils.createHeaders(method, headers, params, this.accessKey, this.secretKey, this.ctUserId); + } + + public String buildUrl(String apiUrl) { + return PROTOCOL + apiUrl; + } + + public Result doGet(String url, Map headers) { + HttpClient httpClient = new HttpClient(20 * 1000, PostDataFormat.RAW); + httpClient.setDataFormat(null); + return httpClient.get(buildHeader(HttpRequestMethod.GET, headers, null), null, buildUrl(url)); + } + + public Result doGet(String url, Map headers, Map params) { + HttpClient httpClient = new HttpClient(60 * 1000, PostDataFormat.RAW); + httpClient.setDataFormat(null); + + String uri = buildUrl(url); + Map header = buildHeader(HttpRequestMethod.GET, headers, params); + printCurl(HttpRequestMethod.GET, uri, header, params); + return httpClient.get(header, params, uri); + } + + public Result doPost(String url, Map params) { + HttpClient httpClient = new HttpClient(60 * 1000, PostDataFormat.RAW); + Map header = buildHeader(HttpRequestMethod.POST, null, params); + printCurl(HttpRequestMethod.POST, buildUrl(url), header, params); + return httpClient.post(header, params, buildUrl(url)); + } + + + /** + * 用于请求回调的check + * + * @param result 结果 + * @param title 接口名 + */ + public JSONObject checkResult(Result result, String title) { + Assert.isTrue(result.isSuccess(), "调用" + title + "API失败" + result.getMessage()); + JSONObject object = JSONObject.parseObject(result.getMessage()); + Assert.notNull(object, "调用" + title + "API" + "返回的object为null"); + log.debug("API返回的数据:{}", object.toJSONString()); + Assert.isTrue(object.getInteger("statusCode").equals(SUCCESS_STATUS_CODE), "调用-" + title + "-API-返回的statusCode为" + object.getInteger("statusCode") + "-message:" + object.getString("message") + "-description:" + object.getString("description")); + JSONObject returnObj = object.getJSONObject("returnObj"); + log.debug("returnObj返回的数据:{}", returnObj == null ? null : returnObj.toJSONString()); + return returnObj; + } + + public void printCurl(HttpRequestMethod method, String uri, Map header, Map params) { + StringBuilder builder = new StringBuilder(); + builder.append("curl --location --request"); + if (Objects.requireNonNull(method) == HttpRequestMethod.GET) { + builder.append(" GET ").append("'").append(uri); + + if (params == null || params.isEmpty()) { + builder.append("'"); + } else { + List keyList = new ArrayList<>(params.keySet()); + keyList = keyList.stream().sorted(Comparator.comparing(String::toString)).toList(); + // aa=1&bb=2 + StringBuilder queryBuffer = new StringBuilder(); + for (String key : keyList) { + if (!queryBuffer.isEmpty()) { + queryBuffer.append('&'); + } + Object value = params.get(key); + if (value == null) { + value = ""; + } + String encodedValue = URLEncoder.encode(String.valueOf(value), StandardCharsets.UTF_8); + queryBuffer.append(key).append('=').append(encodedValue); + } + String query = queryBuffer.toString(); + builder.append("?").append(query).append("'"); + } + } else { + builder.append(" POST ").append("'").append(uri); + if (params == null || params.isEmpty()) { + builder.append("'"); + } else { + builder.append("'"); + builder.append(" --data-raw ").append("'{"); + List keyList = new ArrayList<>(params.keySet()); + StringBuilder queryBuffer = new StringBuilder(); + for (String key : keyList) { + if (!queryBuffer.isEmpty()) { + queryBuffer.append(','); + } + Object value = params.get(key); + if (value == null) { + value = ""; + } + String encodedValue = String.valueOf(value); + queryBuffer.append("\"").append(key).append("\"").append(':').append("\"").append(encodedValue).append("\""); + } + builder.append(queryBuffer); + builder.append("}'"); + } + } + + for (Map.Entry entry : header.entrySet()) { + builder.append(" --header ").append("'").append(entry.getKey()).append(":").append(entry.getValue()).append("'"); + } + uri = builder.toString(); + log.debug("uri:{}", uri); + } + + + public List getMetric(List instanceList) { + String url = "/v4.1/monitor/query-vm-latestmetricdata"; + JSONObject body = new JSONObject(); + body.put("regionID", this.regionId); + body.put("deviceUUIDList", instanceList); + + HttpClient httpClient = new HttpClient(60 * 1000, PostDataFormat.RAW); + String uri = buildUrl(url); + Map header = buildHeader(HttpRequestMethod.POST, null, body); + printCurl(HttpRequestMethod.POST, uri, header, body); + Result result = httpClient.post(header, body, uri); + if (JSONObject.parseObject(result.getMessage()).getInteger("statusCode").equals(800)) { + return JSONObject.parseArray(JSONObject.parseObject(result.getMessage()).getJSONObject("returnObj").getJSONArray("result").toJSONString(), JSONObject.class); + } else { + throw new RuntimeException("获取监控数据失败,接口返回 " + JSONObject.toJSONString(result)); + } + } + + + public List instanceList() { + int pageNo = 1; + try { + String apiUrl = "/v4/ecs/instance-list"; + + JSONObject body = new JSONObject(); + body.put("regionID", this.getRegionId()); + body.put("pageNo", pageNo); + body.put("pageSize", this.PAGE_SIZE); + HttpClient httpClient = new HttpClient(60 * 1000, PostDataFormat.RAW); + Map header = buildHeader(HttpRequestMethod.POST, null, body); + printCurl(HttpRequestMethod.POST, buildUrl(apiUrl), header, body); + Result result = httpClient.post(header, body, buildUrl(apiUrl)); + + JSONObject returnObj = checkResult(result, "查询虚拟机列表"); + + List results = JSONArray.parseArray(returnObj.getString("results"), JSONObject.class); + if (ListTool.isEmpty(results)) { + return null; + } + while (returnObj.getInteger("totalCount") > results.size()) { + ++pageNo; + body.put("pageNo", pageNo); + result = doPost(apiUrl, body); + returnObj = checkResult(result, "查询虚拟机列表"); + results.addAll(JSONArray.parseArray(returnObj.getString("results"), JSONObject.class)); + } + return results.stream().map(o -> o.getString("iD")).collect(Collectors.toList()); + } catch (Exception e) { + log.error("查询云主机列表失败 : " + e); + } + return new ArrayList<>(); + } + +} diff --git a/src/main/java/com/bocloud/mcs/common/Authentication.java b/src/main/java/com/bocloud/mcs/common/Authentication.java new file mode 100644 index 0000000..59b43f6 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/common/Authentication.java @@ -0,0 +1,62 @@ +package com.bocloud.mcs.common; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +import java.util.List; + +/** + * 平台认证信息(地域)实体类 + * + * @author fanzhenxi + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@ToString +public class Authentication { + + /** + * 地域集合 + */ + private List regions; + + private String regionId; + + /** + * AK + */ + private String secretKey; + + /** + * SK + */ + private String accessKey; + + /** + * 版本号 + */ + private String version; + + /** + * 判断是否是专有云 + */ + private String address; + private String url; + + /** + * 百度云新增 + */ + private String userId; + + + private String account; + + private String password; + + private String projectId; + + private String domainName; +} diff --git a/src/main/java/com/bocloud/mcs/config/CollectorConfiguration.java b/src/main/java/com/bocloud/mcs/config/CollectorConfiguration.java new file mode 100644 index 0000000..c4af2a5 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/config/CollectorConfiguration.java @@ -0,0 +1,35 @@ +package com.bocloud.mcs.config; + +import com.moandjiezana.toml.Toml; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.io.File; +import java.io.InputStream; + +/** + * @author zhangyf + */ +@Slf4j +@Configuration +public class CollectorConfiguration { + + @Value("${collector.config:}") + private String collectorConfigPath; + + @Bean + public Toml collectorConfig() { + if (StringUtils.isBlank(collectorConfigPath)) { + try (InputStream inputStream = this.getClass().getResourceAsStream("/collector.conf")) { + return new Toml().read(inputStream); + } catch (Exception e) { + log.error("load collector.conf failed!", e); + return null; + } + } + return new Toml().read(new File(collectorConfigPath)); + } +} diff --git a/src/main/java/com/bocloud/mcs/config/ScheduledConfig.java b/src/main/java/com/bocloud/mcs/config/ScheduledConfig.java new file mode 100644 index 0000000..5fa3f0f --- /dev/null +++ b/src/main/java/com/bocloud/mcs/config/ScheduledConfig.java @@ -0,0 +1,24 @@ +package com.bocloud.mcs.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.SchedulingConfigurer; +import org.springframework.scheduling.config.ScheduledTaskRegistrar; + +import java.util.concurrent.Executor; +import java.util.concurrent.ScheduledThreadPoolExecutor; + +@Configuration +public class ScheduledConfig implements SchedulingConfigurer { + + @Override + public void configureTasks(ScheduledTaskRegistrar taskRegistrar) { + taskRegistrar.setScheduler(setTaskExecutors()); + } + + @Bean(destroyMethod = "shutdown") + public Executor setTaskExecutors() { + return new ScheduledThreadPoolExecutor(8); + } + +} diff --git a/src/main/java/com/bocloud/mcs/config/WebInterceptorConfiguration.java b/src/main/java/com/bocloud/mcs/config/WebInterceptorConfiguration.java new file mode 100644 index 0000000..1fb05d9 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/config/WebInterceptorConfiguration.java @@ -0,0 +1,29 @@ +package com.bocloud.mcs.config; + +import com.megatron.framework.core.InterceptorBridge; +import com.megatron.framework.core.config.InterceptorConfiguration; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; + +/** + * 拦截器配置类 + * + * @author wangyu + * @version 1.0 + * @since 2018年3月30日 + */ +@Slf4j +@Component +public class WebInterceptorConfiguration implements InterceptorConfiguration { + + @Override + public InterceptorBridge buildBridge() { + InterceptorBridge bridge = new InterceptorBridge(); + bridge.setInterceptors(new ArrayList<>()); + bridge.setMessageConverters(new ArrayList<>()); + bridge.setResolvers(new ArrayList<>()); + return bridge; + } +} diff --git a/src/main/java/com/bocloud/mcs/controller/MetricController.java b/src/main/java/com/bocloud/mcs/controller/MetricController.java new file mode 100644 index 0000000..de8d72f --- /dev/null +++ b/src/main/java/com/bocloud/mcs/controller/MetricController.java @@ -0,0 +1,45 @@ +package com.bocloud.mcs.controller; + +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.exporter.common.TextFormat; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; + +@RestController +public class MetricController { + private final CollectorRegistry collectorRegistry = CollectorRegistry.defaultRegistry; + + + @Value("${metrics.sample.path:/home/cmp/prometheus/metrics}") + private String path; + + @GetMapping("/metrics") + public void writeRegistry(HttpServletRequest request, HttpServletResponse response) { + try { + response.setContentType("text/plain;charset=UTF-8"); + TextFormat.write004(response.getWriter(), collectorRegistry.filteredMetricFamilySamples(Collections.emptySet())); + } catch (IOException e) { + // This actually never happens since StringWriter::write() doesn't + // throw any IOException + throw new RuntimeException("Writing metrics failed", e); + } + } + + @GetMapping("/sample/metrics") + public void writeRegistry2(HttpServletRequest request, HttpServletResponse response) { + try { + response.setContentType("text/plain;charset=UTF-8"); + response.getWriter().write(FileUtils.readFileToString(new File(path), "utf-8")); + } catch (IOException e) { + throw new RuntimeException("Writing metrics failed", e); + } + } +} diff --git a/src/main/java/com/bocloud/mcs/controller/PrometheusController.java b/src/main/java/com/bocloud/mcs/controller/PrometheusController.java new file mode 100644 index 0000000..8b25200 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/controller/PrometheusController.java @@ -0,0 +1,50 @@ +package com.bocloud.mcs.controller; + +import com.bocloud.mcs.entity.PrometheusHttpConfigBean; +import com.megatron.common.utils.ListTool; +import com.megatron.framework.core.RegistryService; +import com.megatron.framework.core.Service; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.*; + +/** + * Prometheus http服务发现 + * + * @author fanzhenxi + * @version 5.6.0-SNAPSHOT + * @date 2022/2/23 18:23 + */ +@RestController +@Slf4j +public class PrometheusController { + + @Autowired + private Service service; + @Autowired + private RegistryService registry; + + @GetMapping("/targets") + public List targets() { + try { + List prometheusHttpConfigBeans = new ArrayList<>(); + List serviceInstances = registry.getServices(service); + List servers = ListTool.isEmpty(serviceInstances) ? new ArrayList<>() : serviceInstances; + Set targets = new HashSet<>(); + Optional.ofNullable(servers).orElse(new ArrayList<>()).forEach(server -> { + server = server.replaceAll("http://", "").replaceAll("https://", ""); + targets.add(server); + }); + PrometheusHttpConfigBean prometheusHttpConfigBean = new PrometheusHttpConfigBean(targets); + prometheusHttpConfigBeans.add(prometheusHttpConfigBean); + return prometheusHttpConfigBeans; + } catch (Exception e) { + log.error("获取zookeeper列表中节点失败!"); + return null; + } + } + +} diff --git a/src/main/java/com/bocloud/mcs/entity/PrometheusHttpConfigBean.java b/src/main/java/com/bocloud/mcs/entity/PrometheusHttpConfigBean.java new file mode 100644 index 0000000..c24a453 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/entity/PrometheusHttpConfigBean.java @@ -0,0 +1,27 @@ +package com.bocloud.mcs.entity; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +import java.util.Set; + +/** + * Prometheus http服务发现机制 + * + * @author fanzhenxi + * @version 5.6.0-SNAPSHOT + * @date 2022/2/23 17:23 + */ +@Data +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class PrometheusHttpConfigBean { + + @JSONField(name = "targets") + Set targets; + +} diff --git a/src/main/java/com/bocloud/mcs/service/internal/CmpCloudVendorService.java b/src/main/java/com/bocloud/mcs/service/internal/CmpCloudVendorService.java new file mode 100644 index 0000000..0458178 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/service/internal/CmpCloudVendorService.java @@ -0,0 +1,78 @@ +package com.bocloud.mcs.service.internal; + +import com.alibaba.fastjson.JSONObject; +import com.bocloud.cmp.boot.model.BoCloudService; +import com.bocloud.cmp.boot.model.BocloudToken; +import com.bocloud.ims.entity.CloudVendor; +import com.bocloud.ims.entity.KvmHost; +import com.bocloud.ims.enums.CloudProvider; +import com.megatron.common.model.GeneralResult; +import com.megatron.common.model.Pager; +import com.megatron.common.model.Param; +import com.megatron.common.model.Sign; +import com.megatron.common.utils.Common; +import com.megatron.common.utils.MapTools; +import com.megatron.framework.core.CurrentService; +import com.megatron.framework.core.Service; +import com.megatron.framework.http.core.HttpMethod; +import com.megatron.framework.http.core.ServiceFactory; +import com.megatron.framework.http.model.RemoteService; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author zhangyf + */ +@RequiredArgsConstructor +@Component +public class CmpCloudVendorService { + + private final ServiceFactory serviceFactory; + private final CurrentService currentService; + + /** + * 查询已开启告警的平台 + * + * @return 平台列表 + */ + public List list(CloudProvider cloudProvider) { + Map headers = new HashMap<>(4); + headers.put(BocloudToken.INTERNAL_TOKEN, currentService.getToken()); + + List paramList = new ArrayList<>(); + paramList.add(new Param(MapTools.simpleMap("is_alarm", 1), Sign.EQ)); + paramList.add(new Param(MapTools.simpleMap("type", cloudProvider.name()), Sign.EQ)); + RemoteService remoteService = serviceFactory.build(Service.create(String.valueOf(BoCloudService.CMP.getName())), + "/v1/vendors", HttpMethod.GET, headers, new Pager(1, Integer.MAX_VALUE, paramList, null).toMap(false)); + GeneralResult result = remoteService.invoke(); + Assert.isTrue(result.isSuccess(), "list cloud vendor failed, error:" + result.getMessage()); + JSONObject data = (JSONObject) result.getData(); + return data.getJSONArray(Common.ROWS).toJavaList(CloudVendor.class); + } + + /** + * 查询云平台下所有宿主机 + * + * @return + */ + public List listKvmHost(Long vendorId) { + Map headers = new HashMap<>(4); + headers.put(BocloudToken.INTERNAL_TOKEN, currentService.getToken()); + + List paramList = new ArrayList<>(); + paramList.add(new Param(MapTools.simpleMap("vendorId", vendorId), Sign.EQ)); + RemoteService remoteService = serviceFactory.build(Service.create(String.valueOf(BoCloudService.CMP.getName())), "/plugins/kvm/v1/hosts", HttpMethod.GET, headers, + new Pager(1, Integer.MAX_VALUE, paramList, null).toMap(false)); + GeneralResult result = remoteService.invoke(); + Assert.isTrue(result.isSuccess(), "get kvm host list failed , error:" + result.getMessage()); + JSONObject data = (JSONObject) result.getData(); + return data.getJSONArray(Common.ROWS).toJavaList(KvmHost.class); + } + +} diff --git a/src/main/java/com/bocloud/mcs/service/internal/CmsCloudVendorService.java b/src/main/java/com/bocloud/mcs/service/internal/CmsCloudVendorService.java new file mode 100644 index 0000000..cf1d65a --- /dev/null +++ b/src/main/java/com/bocloud/mcs/service/internal/CmsCloudVendorService.java @@ -0,0 +1,54 @@ +package com.bocloud.mcs.service.internal; + +import com.bocloud.cmp.boot.model.BoCloudService; +import com.bocloud.cmp.boot.model.BocloudToken; +import com.megatron.common.model.GeneralResult; +import com.megatron.framework.core.CurrentService; +import com.megatron.framework.core.Service; +import com.megatron.framework.http.core.HttpMethod; +import com.megatron.framework.http.core.ServiceFactory; +import com.megatron.framework.http.model.RemoteService; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; + +@Component +public class CmsCloudVendorService { + + private final ServiceFactory serviceFactory; + private final CurrentService currentService; + + public CmsCloudVendorService(ServiceFactory serviceFactory, CurrentService currentService) { + this.serviceFactory = serviceFactory; + this.currentService = currentService; + } + + /** + * 查询平台的运维账号信息(仅FusionCloud适用) + * + * @return + */ + public GeneralResult detailVendorAuth(Long vendorId) { + Map headers = new HashMap<>(4); + headers.put(BocloudToken.INTERNAL_TOKEN, currentService.getToken()); + + String url = "/v1/vendors/" + vendorId + "/authentications"; + RemoteService remoteService = serviceFactory.build(Service.create(String.valueOf(BoCloudService.CMS.getName())), url, HttpMethod.GET, headers, null); + return remoteService.invoke(); + } + + + /** + * 查询已开启监控的FC平台 + * @return + */ + public GeneralResult listFC() { + Map headers = new HashMap<>(4); + headers.put(BocloudToken.INTERNAL_TOKEN, currentService.getToken()); + RemoteService remoteService = serviceFactory.build(Service.create(String.valueOf(BoCloudService.CMS.getName())), "/v1/vendors/authentications", HttpMethod.GET, headers, + new HashMap<>()); + return remoteService.invoke(); + } + +} diff --git a/src/main/java/com/bocloud/mcs/service/internal/ImsCloudServerService.java b/src/main/java/com/bocloud/mcs/service/internal/ImsCloudServerService.java new file mode 100644 index 0000000..fb927b5 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/service/internal/ImsCloudServerService.java @@ -0,0 +1,68 @@ +package com.bocloud.mcs.service.internal; + +import com.alibaba.fastjson.JSONObject; +import com.bocloud.cmp.boot.model.BoCloudService; +import com.bocloud.cmp.boot.model.BocloudToken; +import com.bocloud.mcs.collector.manageone.cce.common.ContainerCluster; +import com.megatron.common.model.GeneralResult; +import com.megatron.common.model.Pager; +import com.megatron.common.model.Param; +import com.megatron.common.utils.Common; +import com.megatron.common.utils.MapTools; +import com.megatron.framework.core.CurrentService; +import com.megatron.framework.core.Service; +import com.megatron.framework.http.core.ServiceFactory; +import com.megatron.framework.http.model.RemoteService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; +import com.megatron.framework.http.core.HttpMethod; +import com.megatron.common.model.Sign; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +@Component +public class ImsCloudServerService { + + @Autowired + private ServiceFactory serviceFactory; + @Autowired + private CurrentService currentService; + + /** + * 列表查询 + * + * @param pager + * @param token 如果为null,则使用internal_token内部调用查询 + * @return + */ + public GeneralResult list(Pager pager, String token) { + Map headers = new HashMap<>(4); + if (token == null) { + headers.put(BocloudToken.INTERNAL_TOKEN, currentService.getToken()); + } else { + headers.put(Common.TOKEN, token); + } + RemoteService remoteService = serviceFactory.build(Service.create(String.valueOf(BoCloudService.CMP.getName())), "/v1/vms", HttpMethod.GET, headers, pager.toMap(false)); + return remoteService.invoke(); + } + + public List getCceContainer(Long vendorId) { + Map headers = new HashMap<>(4); + headers.put(BocloudToken.INTERNAL_TOKEN, currentService.getToken()); + + List paramList = new ArrayList<>(); + paramList.add(new Param(MapTools.simpleMap("vendorId", vendorId), Sign.EQ)); + RemoteService remoteService = serviceFactory.build(Service.create(String.valueOf(BoCloudService.CMP.getName())), + "/plugins/manageone/v1/containers/clusters", HttpMethod.GET, headers, new Pager(1, Integer.MAX_VALUE, paramList, null).toMap(false)); + GeneralResult result = remoteService.invoke(); + Assert.isTrue(result.isSuccess(), "list manager bms failed, error:" + result.getMessage()); + JSONObject data = (JSONObject) result.getData(); + return data.getJSONArray(Common.ROWS).toJavaList(ContainerCluster.class); + } + +} diff --git a/src/main/java/com/bocloud/mcs/syslog/SysLogAppenderTcpAndUdp.java b/src/main/java/com/bocloud/mcs/syslog/SysLogAppenderTcpAndUdp.java new file mode 100644 index 0000000..781dacd --- /dev/null +++ b/src/main/java/com/bocloud/mcs/syslog/SysLogAppenderTcpAndUdp.java @@ -0,0 +1,15 @@ +package com.bocloud.mcs.syslog; + +import ch.qos.logback.classic.net.SyslogAppender; +import ch.qos.logback.core.net.SyslogOutputStream; + +import java.net.SocketException; +import java.net.UnknownHostException; + +public class SysLogAppenderTcpAndUdp extends SyslogAppender { + + @Override + public SyslogOutputStream createOutputStream() throws SocketException, UnknownHostException { + return new SyslogOutputStreamTcpAndUdp(getSyslogHost(), getPort()); + } +} diff --git a/src/main/java/com/bocloud/mcs/syslog/SysLogLevelConvertor.java b/src/main/java/com/bocloud/mcs/syslog/SysLogLevelConvertor.java new file mode 100644 index 0000000..4341a80 --- /dev/null +++ b/src/main/java/com/bocloud/mcs/syslog/SysLogLevelConvertor.java @@ -0,0 +1,22 @@ +package com.bocloud.mcs.syslog; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.pattern.CompositeConverter; + +import java.util.HashMap; +import java.util.Map; + +public class SysLogLevelConvertor extends CompositeConverter { + + private static final Map MAP = new HashMap<>() {{ + this.put("DEBUG", "7"); + this.put("INFO", "6"); + this.put("WARN", "4"); + this.put("ERROR", "3"); + }}; + + @Override + protected String transform(ILoggingEvent iLoggingEvent, String s) { + return MAP.getOrDefault(iLoggingEvent.getLevel().toString(), iLoggingEvent.getLevel().toString()); + } +} \ No newline at end of file diff --git a/src/main/java/com/bocloud/mcs/syslog/SyslogOutputStreamTcpAndUdp.java b/src/main/java/com/bocloud/mcs/syslog/SyslogOutputStreamTcpAndUdp.java new file mode 100644 index 0000000..52ac0cb --- /dev/null +++ b/src/main/java/com/bocloud/mcs/syslog/SyslogOutputStreamTcpAndUdp.java @@ -0,0 +1,70 @@ +package com.bocloud.mcs.syslog; + +import ch.qos.logback.core.net.SyslogOutputStream; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.net.SocketException; +import java.net.UnknownHostException; + +public class SyslogOutputStreamTcpAndUdp extends SyslogOutputStream { + + private static final int MAX_LEN_TCP = 1024; + + private InetAddress addressTcp; + private Socket socketTcp; + private ByteArrayOutputStream baosTcp = new ByteArrayOutputStream(); + final private int portTcp; + + public SyslogOutputStreamTcpAndUdp(String syslogHost, int port) throws SocketException, UnknownHostException { + super(syslogHost, port); + this.addressTcp = InetAddress.getByName(syslogHost); + this.portTcp = port; + try { + this.socketTcp = new Socket(addressTcp, portTcp); + } catch (IOException e) { + throw new SocketException(e); + } + } + + @Override + public void write(byte[] byteArray, int offset, int len) throws IOException { + super.write(byteArray, offset, len); + baosTcp.write(byteArray, offset, len); + } + + @Override + public void flush() throws IOException { + super.flush(); + + byte[] bytes = baosTcp.toByteArray(); + + // clean up for next round + if (baosTcp.size() > MAX_LEN_TCP) { + baosTcp = new ByteArrayOutputStream(); + } else { + baosTcp.reset(); + } + + // after a failure, it can happen that bytes.length is zero + // in that case, there is no point in sending out an empty message/ + if (bytes.length == 0) { + return; + } + if (this.socketTcp != null) { + socketTcp.getOutputStream().write(bytes); + } + } + + @Override + public void close() { + super.close(); + addressTcp = null; + try { + socketTcp.close(); + } catch (IOException ignored) { + } + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..d5f8865 --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,111 @@ +server: + port: '18080' +spring: + application: + name: mcs + thread: + core: + pool: + size: 1000 + max: + pool: + size: 1000 + queue: + capacity: 2000 + upload: + path: /home/cmp + cloud: + zookeeper: + connect-string: 10.20.12.88:2181 + home: /bocloud/cmp/product + auth: + username: bocloud + password: AES@CoU6oY/zuAoBbu6cdgOKlA== + config: + enabled: false + discovery: + register: true + enable: true + instance-host: 10.40.51.188 + instance-port: ${server.port} + root: ${spring.cloud.zookeeper.home}/services + conn: + timeout: '50000' + session: + timeout: '50000' + datasource: + druid: + dialect: mysql + url: jdbc:mysql://10.20.12.88:3306/cmp?characterEncoding=utf8&useSSL=false + username: cmp + password: Gy2VghUgWHszx8gLFT4etT9ZVOukJi73KoG1q3Oz/DAz5h2mFVVunjcyaaKT9tMPsBgoWBFRmrYbhgqJqT/Q8A== + initialSize: '2' + maxActive: '30' + minIdle: '2' + maxWait: '60000' + poolPreparedStatements: 'true' + maxOpenPreparedStatements: '20' + maxPoolPreparedStatementPerConnectionSize: '20' + validationQuery: select 1 + validationQueryTimeout: '1' + testOnBorrow: 'true' + testOnReturn: 'true' + testWhileIdle: 'true' + remove-abandoned: 'true' + remove-abandoned-timeout: '60' + log-abandoned: 'true' + timeBetweenEvictionRunsMillis: '10000' + minEvictableIdleTimeMillis: '30001' + asyncCloseConnectionEnable: 'true' + filters: config,stat,wall,log4j + publicKey: MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAJfVJOpXYGy8aBCk3zEoKKQDTVCCvJwhhithfY/I5PUvzFBAYygunmuCtUhzPUZ+1RJQds0Q4fu07m5mr5kv5ocCAwEAAQ== + connectionProperties: config.decrypt=true;config.decrypt.key=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAJfVJOpXYGy8aBCk3zEoKKQDTVCCvJwhhithfY/I5PUvzFBAYygunmuCtUhzPUZ+1RJQds0Q4fu07m5mr5kv5ocCAwEAAQ==;druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 + use-global-data-source-stat: 'true' + rabbitmq: + host: 10.20.12.88 + port: 5672 + username: admin + password: AES@CoU6oY/zuAoBbu6cdgOKlA== + virtual-host: cmp + dynamic: 'true' + banner: + charset: UTF-8 + freemarker: + checkTemplateLocation: 'false' + data: + redis: + # standalone + host: 10.20.12.88 + port: 6379 + timeout: 60000ms + password: AES@CoU6oY/zuAoBbu6cdgOKlA== + lettuce: + pool: + max-active: '1024' + max-idle: '16' + min-idle: '4' + max-wait: -1ms + servlet: + multipart: + max-file-size: 2MB + max-request-size: 20MB + main: + allow-circular-references: true +logging: + config: classpath:logback-spring.xml + dir: /log/services + level: + root: info + com: + bocloud: info + file: + max-size: 100MB + max-history: '30' + total-size-cap: 2GB + syslog: + #host: mai-miao.cn + #port: 8340 + host: 10.20.12.88 + port: 50000 + facility: LOCAL3 + level: info \ No newline at end of file diff --git a/src/main/resources/banner.txt b/src/main/resources/banner.txt new file mode 100644 index 0000000..c75f76f --- /dev/null +++ b/src/main/resources/banner.txt @@ -0,0 +1,11 @@ +$$$$$$$\ $$\ $$$$$$\ $$\ $$\ $$$$$$$\ $$\ $$\ $$$$$$\ $$$$$$\ +$$ __$$\ $$ |$$ __$$\ $$$\ $$$ |$$ __$$\ $$$\ $$$ |$$ __$$\ $$ __$$\ +$$ | $$ | $$$$$$\ $$\ $$\ $$$$$$\ $$$$$$$\ $$$$$$$ |$$ / \__|$$$$\ $$$$ |$$ | $$ | $$$$\ $$$$ |$$ / \__|$$ / \__| +$$$$$$$\ |$$ __$$\ $$ | $$ |$$ __$$\ $$ __$$\ $$ __$$ |$$ | $$\$$\$$ $$ |$$$$$$$ | $$\$$\$$ $$ |$$ | \$$$$$$\ +$$ __$$\ $$$$$$$$ |$$ | $$ |$$ / $$ |$$ | $$ |$$ / $$ |$$ | $$ \$$$ $$ |$$ ____/ $$ \$$$ $$ |$$ | \____$$\ +$$ | $$ |$$ ____|$$ | $$ |$$ | $$ |$$ | $$ |$$ | $$ |$$ | $$\ $$ |\$ /$$ |$$ | $$ |\$ /$$ |$$ | $$\ $$\ $$ | +$$$$$$$ |\$$$$$$$\ \$$$$$$$ |\$$$$$$ |$$ | $$ |\$$$$$$$ |\$$$$$$ |$$ | \_/ $$ |$$ | $$ | \_/ $$ |\$$$$$$ |\$$$$$$ | +\_______/ \_______| \____$$ | \______/ \__| \__| \_______| \______/ \__| \__|\__| \__| \__| \______/ \______/ + $$\ $$ | + \$$$$$$ | + \______/ \ No newline at end of file diff --git a/src/main/resources/collector.conf b/src/main/resources/collector.conf new file mode 100644 index 0000000..d9c2695 --- /dev/null +++ b/src/main/resources/collector.conf @@ -0,0 +1,20 @@ +[aliyun] +[aliyun.ecs] + metric_include = ["cpu_total_average","memory_usedutilization_average","load_1m_average","load_5m_average","load_15m_average","diskusage_utilization_average", +"disk_writebytes_average","disk_readbytes_average","disk_writeiops_average","disk_readiops_average","fs_inodeutilization_average","net_tcpconnection_average", +"CPUUtilization_average","IntranetInRate_average","InternetInRate_average","InternetOutRate_average","IntranetOutRate_average","DiskReadBPS_average","DiskWriteBPS_average", +"DiskReadIOPS_average","DiskWriteIOPS_average"] +[aliyun.rds] +[aliyun.redis] +[aliyun.mongodb] +[aliyun.oceanbase] +[aliyun.kafka] +[aliyun.rocketmq] +[apsarastack.ecs] + metric_include = ["cpu_total_average","memory_usedutilization_average","load_1m_average","load_5m_average","load_15m_average","diskusage_utilization_average", +"disk_writebytes_average","disk_readbytes_average","disk_writeiops_average","disk_readiops_average","fs_inodeutilization_average","net_tcpconnection_average", +"CPUUtilization_average","IntranetInRate_average","InternetInRate_average","InternetOutRate_average","IntranetOutRate_average","DiskReadBPS_average","DiskWriteBPS_average", +"DiskReadIOPS_average","DiskWriteIOPS_average"] +[apsarastack.rds] + metric_include = ["DiskUsage_average","IOPSUsage_average","ConnectionUsage_average","CpuUsage_average","MemoryUsage_average","MySQL_NetworkInNew_average", +"MySQL_NetworkOutNew_average","MySQL_QPS_average","DockerCpuUsage_average"] \ No newline at end of file diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..a46e42b --- /dev/null +++ b/src/main/resources/logback-spring.xml @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + ${local_log_pattern} + + + + + + ${logging.dir}/${service.name}.log + + ${logging.dir}/history/${service.name}.%d{yyyy-MM-dd}.%i.log.gz + + + ${logging.file.max-size} + + ${logging.file.max-history} + ${logging.file.total-size-cap} + + + ${local_log_pattern} + + + + + + ${logging.syslog.host} + ${logging.syslog.port} + ${logging.syslog.facility} + ${sys_log_pattern} + + ${logging.syslog.level} + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file