refactor

README.md

@@ -9,7 +9,7 @@ enterprise-level websites monitoring system
 - customize header
 - customize post body
 - support get post put method
-- send to open-falcon
+- send to open-falcon、statsd、prometheus
 
 ## Architecture
 ![Architecture](img/urlooker_arch.png)
@@ -28,7 +28,6 @@ enterprise-level websites monitoring system
 ## Install
 ##### dependence
 ```
-yum install -y redis
 yum install -y mysql-server
 ```
 ##### import mysql database

configs/agent.yml

@@ -1,5 +1,4 @@
-debug: false
-hostname: "127.0.0.1"
+idc: "default"
 worker: 1000
 web:
   addrs: ["127.0.0.1:1985"]

configs/alarm.yml

@@ -1,30 +1,16 @@
-remain: 10
 rpc:
     listen: "0.0.0.0:1986"
 web: 
-    addrs: ["127.0.0.1:1985"]
-    timeout: 300
-    interval: 60
-alarm: 
-    enabled: true
-    minInterval: 180
-    queuePattern: "event"
-    redis: 
-        dsn: "127.0.0.1:6379"
-        maxIdle: 5
-        connTimeout: 20000
-        readTimeout: 20000
-        writeTimeout: 20000
-queue: 
-    sms: "/sms"
-    mail: "/mail"
-worker: 
-    sms: 10
-    mail: 50
-sms: ""
+    addrs: 
+      - "127.0.0.1:1985"
+    timeout: 300 
+    interval: 60 #get strategy interval
 smtp: 
     addr: "mail.addr:25"
     username: "[email protected]"
     password: ""
     from: "[email protected]"
-
+worker: 
+    sms: 10  #send sms concurrency
+    mail: 50 #send mail concurrency
+# remain: 30

configs/web.yml

@@ -3,7 +3,8 @@ salt: "e84d0755dda7a538315ea7a95a8c36af"
 register: true #如果不想其他人注册，可以设置为false
 showDurationMin: 30 #展示曲线的时间范围，单位为分钟
 keepDurationHour: 24  #api探测结果在数据库保存的时间范围，单位为小时
-dns: "114.114.114.114:53" #填写你们域名使用的dns服务
+idc:
+    - "default" #将探测目标分配给哪个机房，agent组件也有一个对应的配置
 http:
     listen: "0.0.0.0:1984"
     secret: "secret"
@@ -32,12 +33,15 @@ alarm: #告警判断组件，为true表示使用alarm组件
     sleepTime: 30
     cluster:
         node-1: "127.0.0.1:1986"
-monitorMap:
-    default: ["127.0.0.1"] #数组中的字符串需要和agent所在机器的hostname相同
 falcon: #open-falcon接收数据的地址，为true表示要向falcon推送数据
     enable: false
     addr: "http://falcon.transfer.addr/api/push"
     interval: 60
-internalDns: #如果为true，表示使用自己的dns解析来解析域名
+statsd:
     enable: false
-    cmd: "./script/dsn.sh"
+    addr: "ip:port"
+prom: 
+    enable: false
+    addr: "ip:port"
+
+

dataobj/event.go

@@ -1,7 +1,5 @@
 package dataobj
 
-import "fmt"
-
 type Event struct {
 	Id          int64  `json:"id"`
 	EventId     string `json:"event_id"`
@@ -17,19 +15,17 @@ type Event struct {
 	MaxStep     int    `json:"max_step"`
 }
 
-func (this *Event) String() string {
-	return fmt.Sprintf(
-		"<Id:%s, EventId:,%s Ip:%s, Url:%s, EventTime:%v, StrategyId:%d, RespTime:%s, RespCode:%s, Status:%s, (%d/%d)>",
-		this.Id,
-		this.EventId,
-		this.Ip,
-		this.Url,
-		this.EventTime,
-		this.StrategyId,
-		this.RespTime,
-		this.RespCode,
-		this.Status,
-		this.CurrentStep,
-		this.MaxStep,
-	)
+type EventQueue struct {
+	Id          int64  `json:"id"`
+	EventId     string `json:"event_id"`
+	Status      string `json:"status"`
+	Url         string `json:"url"`
+	Ip          string `json:"ip"`
+	EventTime   int64  `json:"event_time"`
+	StrategyId  int64  `json:"strategy_id"`
+	RespTime    int    `json:"resp_time"`
+	RespCode    string `json:"resp_code"`
+	Result      int64  `json:"result"`
+	CurrentStep int    `json:"current_step"`
+	MaxStep     int    `json:"max_step"`
 }

dataobj/item.go

@@ -6,18 +6,12 @@ import (
 	"io"
 )
 
-type IpIdc struct {
-	Ip  string
-	Idc string
-}
-
 //下发给agent的数据结构
 type DetectedItem struct {
 	Sid        int64  `json:"sid"`
 	Method     string `json:"method"`
 	Domain     string `json:"domain"`
 	Target     string `json:"target"`
-	Ip         string `json:"ip"`
 	Keywords   string `json:"keywords"`
 	Timeout    int    `json:"timeout"`
 	Creator    string `json:"creator"`
@@ -42,13 +36,13 @@ type CheckResult struct {
 	RespTime int    `json:"resp_time"`
 	Status   int64  `json:"status"`
 	PushTime int64  `json:"push_time"`
-	Ip       string `json:"ip"`
+	Step     int64  `json:"step"`
 }
 
 type ItemStatus struct {
 	Id       int64  `json:"id"`
 	Sid      int64  `json:"sid"`
-	Ip       string `json:"ip"`
+	Ip       string `json:"ip"` //agent所在机器的ip
 	RespTime int    `json:"resp_time"`
 	RespCode string `json:"resp_code"`
 	PushTime int64  `json:"push_time"`
@@ -63,7 +57,7 @@ func (this *ItemStatus) PK() string {
 }
 
 type SendResultReq struct {
-	Hostname     string
+	Ip           string
 	CheckResults []*CheckResult
 }
 

dataobj/stra.go

@@ -2,6 +2,7 @@ package dataobj
 
 type Strategy struct {
 	Id         int64  `json:"id"`
+	Idc        string `json:"idc"`
 	Method     string `json:"method"`
 	Url        string `json:"url"`
 	Enable     int    `json:"enable"`

go.mod

@@ -5,14 +5,15 @@ go 1.12
 require (
 	github.com/astaxie/beego v1.12.0
 	github.com/codegangsta/negroni v1.0.0
-	github.com/garyburd/redigo v1.6.0
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/go-xorm/xorm v0.7.9
 	github.com/gorilla/context v1.1.1
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/securecookie v1.1.1
-	github.com/miekg/dns v1.1.26
+	github.com/kr/pretty v0.2.0 // indirect
 	github.com/mmitton/ldap v0.0.0-20170928125358-2890b18948f2
+	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
+	github.com/quipo/statsd v0.0.0-20180118161217-3d6a5565f314
 	github.com/sirupsen/logrus v1.4.2
 	github.com/toolkits/container v0.0.0-20151219225805-ba7d73adeaca
 	github.com/toolkits/core v0.0.0-20141116054942-0ebf14900fe2 // indirect
@@ -23,8 +24,10 @@ require (
 	github.com/toolkits/sys v0.0.0-20170615103026-1f33b217ffaf
 	github.com/toolkits/web v0.0.0-20160312232617-dc0f03327e1d
 	github.com/unrolled/render v1.0.1
+	golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 // indirect
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
+	golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe // indirect
 	gopkg.in/asn1-ber.v1 v1.0.0-00010101000000-000000000000 // indirect
-	gopkg.in/ldap.v2 v2.5.1 // indirect
 	gopkg.in/yaml.v2 v2.2.7
 	stathat.com/c/consistent v1.0.0
 )