586 lines
16 KiB
Go
586 lines
16 KiB
Go
/*
|
||
* @Author: haodaquan
|
||
* @Date: 2017-06-21 12:56:08
|
||
* @Last Modified by: Bee
|
||
* @Last Modified time: 2019-02-17 22:10:15
|
||
*/
|
||
|
||
package jobs
|
||
|
||
import (
|
||
"bytes"
|
||
"fmt"
|
||
"io/ioutil"
|
||
"net"
|
||
"os/exec"
|
||
"runtime/debug"
|
||
"time"
|
||
|
||
"runtime"
|
||
"strconv"
|
||
"strings"
|
||
|
||
"encoding/json"
|
||
"github.com/astaxie/beego"
|
||
"github.com/axgle/mahonia"
|
||
"github.com/george518/PPGo_Job/models"
|
||
"github.com/george518/PPGo_Job/notify"
|
||
"github.com/linxiaozhi/go-telnet"
|
||
"github.com/pkg/errors"
|
||
"golang.org/x/crypto/ssh"
|
||
)
|
||
|
||
type Job struct {
|
||
jobKey int // jobId = id*10000+serverId
|
||
id int // taskID
|
||
logId int64 // 日志记录ID
|
||
serverId int //服务器信息
|
||
serverName string //服务器名称
|
||
name string // 任务名称
|
||
task *models.Task // 任务对象
|
||
runFunc func(time.Duration) (string, string, error, bool) // 执行函数
|
||
status int // 任务状态，大于0表示正在执行中
|
||
Concurrent bool // 同一个任务是否允许并行执行
|
||
}
|
||
|
||
func NewJobFromTask(task *models.Task) ([]*Job, error) {
|
||
if task.Id < 1 {
|
||
return nil, fmt.Errorf("ToJob: 缺少id")
|
||
}
|
||
|
||
if task.ServerIds == "" {
|
||
return nil, fmt.Errorf("任务执行失败，找不到执行的服务器")
|
||
}
|
||
|
||
TaskServerIdsArr := strings.Split(task.ServerIds, ",")
|
||
|
||
jobArr := make([]*Job, 0)
|
||
|
||
for _, server_id := range TaskServerIdsArr {
|
||
if server_id == "0" {
|
||
//本地执行
|
||
job := NewCommandJob(task.Id, 0, task.TaskName, task.Command)
|
||
job.task = task
|
||
job.Concurrent = task.Concurrent == 1
|
||
job.serverId = 0
|
||
job.serverName = "本地服务器"
|
||
jobArr = append(jobArr, job)
|
||
} else {
|
||
server_id_int, _ := strconv.Atoi(server_id)
|
||
//远程执行
|
||
server, _ := models.TaskServerGetById(server_id_int)
|
||
|
||
if server.Status == 1 {
|
||
fmt.Println("服务器已禁用")
|
||
continue
|
||
}
|
||
|
||
if server.ConnectionType == 0 {
|
||
if server.Type == 0 {
|
||
//密码验证登录服务器
|
||
job := RemoteCommandJobByPassword(task.Id, server_id_int, task.TaskName, task.Command, server)
|
||
job.task = task
|
||
job.Concurrent = task.Concurrent == 1
|
||
job.serverId = server_id_int
|
||
job.serverName = server.ServerName
|
||
jobArr = append(jobArr, job)
|
||
} else {
|
||
job := RemoteCommandJob(task.Id, server_id_int, task.TaskName, task.Command, server)
|
||
job.task = task
|
||
job.Concurrent = task.Concurrent == 1
|
||
job.serverId = server_id_int
|
||
job.serverName = server.ServerName
|
||
jobArr = append(jobArr, job)
|
||
}
|
||
} else if server.ConnectionType == 1 {
|
||
if server.Type == 0 {
|
||
//密码验证登录服务器
|
||
job := RemoteCommandJobByTelnetPassword(task.Id, server_id_int, task.TaskName, task.Command, server)
|
||
job.task = task
|
||
job.Concurrent = task.Concurrent == 1
|
||
job.serverId = server_id_int
|
||
job.serverName = server.ServerName
|
||
jobArr = append(jobArr, job)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return jobArr, nil
|
||
}
|
||
|
||
func NewCommandJob(id int, serverId int, name string, command string) *Job {
|
||
job := &Job{
|
||
id: id,
|
||
name: name,
|
||
}
|
||
|
||
job.jobKey = jobKey(id, serverId)
|
||
job.runFunc = func(timeout time.Duration) (string, string, error, bool) {
|
||
bufOut := new(bytes.Buffer)
|
||
bufErr := new(bytes.Buffer)
|
||
//cmd := exec.Command("/bin/bash", "-c", command)
|
||
var cmd *exec.Cmd
|
||
if runtime.GOOS == "windows" {
|
||
cmd = exec.Command("CMD", "/C", command)
|
||
} else {
|
||
cmd = exec.Command("sh", "-c", command)
|
||
}
|
||
cmd.Stdout = bufOut
|
||
cmd.Stderr = bufErr
|
||
cmd.Start()
|
||
err, isTimeout := runCmdWithTimeout(cmd, timeout)
|
||
|
||
return gbkAsUtf8(bufOut.String()), gbkAsUtf8(bufErr.String()), err, isTimeout
|
||
}
|
||
return job
|
||
}
|
||
|
||
//远程执行任务 密钥验证
|
||
func RemoteCommandJob(id int, serverId int, name string, command string, servers *models.TaskServer) *Job {
|
||
job := &Job{
|
||
id: id,
|
||
name: name,
|
||
serverId: serverId,
|
||
}
|
||
|
||
job.jobKey = jobKey(id, serverId)
|
||
|
||
job.runFunc = func(timeout time.Duration) (string, string, error, bool) {
|
||
|
||
key, err := ioutil.ReadFile(servers.PrivateKeySrc)
|
||
if err != nil {
|
||
return "", "", err, false
|
||
}
|
||
// Create the Signer for this private key.
|
||
signer, err := ssh.ParsePrivateKey(key)
|
||
if err != nil {
|
||
return "", "", err, false
|
||
}
|
||
addr := fmt.Sprintf("%s:%d", servers.ServerIp, servers.Port)
|
||
config := &ssh.ClientConfig{
|
||
User: servers.ServerAccount,
|
||
Auth: []ssh.AuthMethod{
|
||
// Use the PublicKeys method for remote authentication.
|
||
ssh.PublicKeys(signer),
|
||
},
|
||
//HostKeyCallback: ssh.FixedHostKey(hostKey),
|
||
HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
|
||
return nil
|
||
},
|
||
}
|
||
// Connect to the remote server and perform the SSH handshake.47.93.220.5
|
||
client, err := ssh.Dial("tcp", addr, config)
|
||
if err != nil {
|
||
return "", "", err, false
|
||
}
|
||
|
||
defer client.Close()
|
||
|
||
session, err := client.NewSession()
|
||
if err != nil {
|
||
return "", "", err, false
|
||
}
|
||
defer session.Close()
|
||
|
||
// Once a Session is created, you can execute a single command on
|
||
// the remote side using the Run method.
|
||
|
||
var b bytes.Buffer
|
||
var c bytes.Buffer
|
||
session.Stdout = &b
|
||
session.Stderr = &c
|
||
|
||
//session.Output(command)
|
||
if err := session.Run(command); err != nil {
|
||
return "", "", err, false
|
||
}
|
||
isTimeout := false
|
||
return b.String(), c.String(), err, isTimeout
|
||
}
|
||
return job
|
||
}
|
||
|
||
func RemoteCommandJobByPassword(id int, serverId int, name string, command string, servers *models.TaskServer) *Job {
|
||
var (
|
||
auth []ssh.AuthMethod
|
||
addr string
|
||
clientConfig *ssh.ClientConfig
|
||
client *ssh.Client
|
||
session *ssh.Session
|
||
err error
|
||
)
|
||
|
||
job := &Job{
|
||
id: id,
|
||
name: name,
|
||
serverId: serverId,
|
||
}
|
||
job.jobKey = jobKey(id, serverId)
|
||
job.runFunc = func(timeout time.Duration) (string, string, error, bool) {
|
||
// get auth method
|
||
auth = make([]ssh.AuthMethod, 0)
|
||
auth = append(auth, ssh.Password(servers.Password))
|
||
|
||
clientConfig = &ssh.ClientConfig{
|
||
User: servers.ServerAccount,
|
||
Auth: auth,
|
||
HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
|
||
return nil
|
||
},
|
||
//Timeout: 1000 * time.Second,
|
||
}
|
||
|
||
// connet to ssh
|
||
addr = fmt.Sprintf("%s:%d", servers.ServerIp, servers.Port)
|
||
|
||
if client, err = ssh.Dial("tcp", addr, clientConfig); err != nil {
|
||
return "", "", err, false
|
||
}
|
||
|
||
defer client.Close()
|
||
|
||
// create session
|
||
if session, err = client.NewSession(); err != nil {
|
||
return "", "", err, false
|
||
}
|
||
|
||
var b bytes.Buffer
|
||
var c bytes.Buffer
|
||
session.Stdout = &b
|
||
session.Stderr = &c
|
||
//session.Output(command)
|
||
if err := session.Run(command); err != nil {
|
||
return "", "", err, false
|
||
}
|
||
isTimeout := false
|
||
return b.String(), c.String(), err, isTimeout
|
||
}
|
||
|
||
return job
|
||
}
|
||
|
||
func RemoteCommandJobByTelnetPassword(id int, serverId int, name string, command string, servers *models.TaskServer) *Job {
|
||
|
||
job := &Job{
|
||
id: id,
|
||
name: name,
|
||
serverId: serverId,
|
||
}
|
||
|
||
job.jobKey = jobKey(id, serverId)
|
||
job.runFunc = func(timeout time.Duration) (string, string, error, bool) {
|
||
|
||
addr := fmt.Sprintf("%s:%d", servers.ServerIp, servers.Port)
|
||
conn, err := gote.DialTimeout("tcp", addr, timeout)
|
||
if err != nil {
|
||
return "", "", err, false
|
||
}
|
||
|
||
defer conn.Close()
|
||
|
||
buf := make([]byte, 4096)
|
||
|
||
if _, err = conn.Read(buf); err != nil {
|
||
return "", "", err, false
|
||
}
|
||
|
||
if _, err = conn.Write([]byte(servers.ServerAccount + "\r\n")); err != nil {
|
||
return "", "", err, false
|
||
}
|
||
|
||
if _, err = conn.Read(buf); err != nil {
|
||
return "", "", err, false
|
||
}
|
||
|
||
if _, err = conn.Write([]byte(servers.Password + "\r\n")); err != nil {
|
||
return "", "", err, false
|
||
}
|
||
|
||
if _, err = conn.Read(buf); err != nil {
|
||
return "", "", err, false
|
||
}
|
||
|
||
loginStr := gbkAsUtf8(string(buf[:]))
|
||
if !strings.Contains(loginStr, ">") {
|
||
return "", "", errors.Errorf("Login failed!"), false
|
||
}
|
||
|
||
commandArr := strings.Split(command, "\n")
|
||
|
||
out, n := "", 0
|
||
for _, c := range commandArr {
|
||
_, err = conn.Write([]byte(c + "\r\n"))
|
||
if err != nil {
|
||
return "", "", err, false
|
||
}
|
||
|
||
n, err = conn.Read(buf)
|
||
|
||
out = out + gbkAsUtf8(string(buf[0:n]))
|
||
if err != nil ||
|
||
strings.Contains(out, "'"+c+"' is not recognized as an internal or external command") ||
|
||
strings.Contains(out, "'"+c+"' 不是内部或外部命令，也不是可运行的程序") {
|
||
return out, "", fmt.Errorf(gbkAsUtf8(string(buf[0:n]))), false
|
||
}
|
||
}
|
||
|
||
return out, "", nil, false
|
||
}
|
||
|
||
return job
|
||
}
|
||
|
||
func (j *Job) Status() int {
|
||
return j.status
|
||
}
|
||
|
||
func (j *Job) GetName() string {
|
||
return j.name
|
||
}
|
||
|
||
func (j *Job) GetId() int {
|
||
return j.id
|
||
}
|
||
|
||
func (j *Job) GetLogId() int64 {
|
||
return j.logId
|
||
}
|
||
|
||
func (j *Job) Run() {
|
||
if !j.Concurrent && j.status > 0 {
|
||
beego.Warn(fmt.Sprintf("任务[%d]上一次执行尚未结束，本次被忽略。", j.jobKey))
|
||
return
|
||
}
|
||
|
||
defer func() {
|
||
if err := recover(); err != nil {
|
||
beego.Error(err, "\n", string(debug.Stack()))
|
||
}
|
||
}()
|
||
|
||
if workPool != nil {
|
||
workPool <- true
|
||
defer func() {
|
||
<-workPool
|
||
}()
|
||
}
|
||
|
||
beego.Debug(fmt.Sprintf("开始执行任务: %d", j.jobKey))
|
||
|
||
j.status++
|
||
defer func() {
|
||
j.status--
|
||
}()
|
||
|
||
t := time.Now()
|
||
timeout := time.Duration(time.Hour * 24)
|
||
if j.task.Timeout > 0 {
|
||
timeout = time.Second * time.Duration(j.task.Timeout)
|
||
}
|
||
cmdOut, cmdErr, err, isTimeout := j.runFunc(timeout)
|
||
ut := time.Now().Sub(t) / time.Millisecond
|
||
|
||
// 插入日志
|
||
log := new(models.TaskLog)
|
||
log.TaskId = j.id
|
||
log.ServerId = j.serverId
|
||
log.ServerName = j.serverName
|
||
log.Output = cmdOut
|
||
log.Error = cmdErr
|
||
log.ProcessTime = int(ut)
|
||
log.CreateTime = t.Unix()
|
||
|
||
if isTimeout {
|
||
log.Status = models.TASK_TIMEOUT
|
||
log.Error = fmt.Sprintf("任务执行超过 %d 秒\n----------------------\n%s\n", int(timeout/time.Second), cmdErr)
|
||
} else if err != nil {
|
||
log.Status = models.TASK_ERROR
|
||
log.Error = err.Error() + ":" + cmdErr
|
||
}
|
||
|
||
if log.Status < 0 && j.task.IsNotify == 1 {
|
||
if j.task.NotifyUserIds != "0" && j.task.NotifyUserIds != "" {
|
||
adminInfo := AllAdminInfo(j.task.NotifyUserIds)
|
||
phone := make(map[string]string, 0)
|
||
dingtalk := make(map[string]string, 0)
|
||
wechat := make(map[string]string, 0)
|
||
toEmail := ""
|
||
for _, v := range adminInfo {
|
||
if v.Phone != "0" && v.Phone != "" {
|
||
phone[v.Phone] = v.Phone
|
||
}
|
||
if v.Email != "0" && v.Email != "" {
|
||
toEmail += v.Email + ";"
|
||
}
|
||
if v.Dingtalk != "0" && v.Dingtalk != "" {
|
||
dingtalk[v.Dingtalk] = v.Dingtalk
|
||
}
|
||
if v.Wechat != "0" && v.Wechat != "" {
|
||
wechat[v.Wechat] = v.Wechat
|
||
}
|
||
}
|
||
toEmail = strings.TrimRight(toEmail, ";")
|
||
|
||
TextStatus := []string{
|
||
"超时",
|
||
"错误",
|
||
"正常",
|
||
}
|
||
|
||
status := log.Status + 2
|
||
|
||
title, content, taskOutput, errOutput := "", "", "", ""
|
||
|
||
notifyTpl, err := models.NotifyTplGetById(j.task.NotifyTplId)
|
||
if err != nil {
|
||
notifyTpl, err := models.NotifyTplGetByTplType(j.task.NotifyType, models.NotifyTplTypeSystem)
|
||
if err == nil {
|
||
title = notifyTpl.Title
|
||
content = notifyTpl.Content
|
||
}
|
||
} else {
|
||
title = notifyTpl.Title
|
||
content = notifyTpl.Content
|
||
}
|
||
|
||
taskOutput = strings.Replace(log.Output, "\n", " ", -1)
|
||
taskOutput = strings.Replace(taskOutput, "\"", "\\\"", -1)
|
||
errOutput = strings.Replace(log.Error, "\n", " ", -1)
|
||
errOutput = strings.Replace(errOutput, "\"", "\\\"", -1)
|
||
|
||
if title != "" {
|
||
title = strings.Replace(title, "{{TaskId}}", strconv.Itoa(j.task.Id), -1)
|
||
title = strings.Replace(title, "{{TaskName}}", j.task.TaskName, -1)
|
||
title = strings.Replace(title, "{{ExecuteCommand}}", j.task.Command, -1)
|
||
title = strings.Replace(title, "{{ExecuteTime}}", beego.Date(time.Unix(log.CreateTime, 0), "Y-m-d H:i:s"), -1)
|
||
title = strings.Replace(title, "{{ProcessTime}}", strconv.FormatFloat(float64(log.ProcessTime)/1000, 'f', 6, 64), -1)
|
||
title = strings.Replace(title, "{{ExecuteStatus}}", TextStatus[status], -1)
|
||
title = strings.Replace(title, "{{TaskOutput}}", taskOutput, -1)
|
||
title = strings.Replace(title, "{{ErrorOutput}}", errOutput, -1)
|
||
}
|
||
|
||
if content != "" {
|
||
content = strings.Replace(content, "{{TaskId}}", strconv.Itoa(j.task.Id), -1)
|
||
content = strings.Replace(content, "{{TaskName}}", j.task.TaskName, -1)
|
||
content = strings.Replace(content, "{{ExecuteCommand}}", j.task.Command, -1)
|
||
content = strings.Replace(content, "{{ExecuteTime}}", beego.Date(time.Unix(log.CreateTime, 0), "Y-m-d H:i:s"), -1)
|
||
content = strings.Replace(content, "{{ProcessTime}}", strconv.FormatFloat(float64(log.ProcessTime)/1000, 'f', 6, 64), -1)
|
||
content = strings.Replace(content, "{{ExecuteStatus}}", TextStatus[status], -1)
|
||
content = strings.Replace(content, "{{TaskOutput}}", taskOutput, -1)
|
||
content = strings.Replace(content, "{{ErrorOutput}}", errOutput, -1)
|
||
}
|
||
|
||
if j.task.NotifyType == 0 && toEmail != "" {
|
||
//邮件
|
||
mailtype := "html"
|
||
ok := notify.SendToChan(toEmail, title, content, mailtype)
|
||
if !ok {
|
||
fmt.Println("发送邮件错误", toEmail)
|
||
}
|
||
} else if j.task.NotifyType == 1 && len(phone) > 0 {
|
||
//信息
|
||
param := make(map[string]string)
|
||
err := json.Unmarshal([]byte(content), ¶m)
|
||
if err != nil {
|
||
fmt.Println("发送信息错误", err)
|
||
return
|
||
}
|
||
|
||
ok := notify.SendSmsToChan(phone, param)
|
||
if !ok {
|
||
fmt.Println("发送信息错误", phone)
|
||
}
|
||
} else if j.task.NotifyType == 2 && len(dingtalk) > 0 {
|
||
//钉钉
|
||
param := make(map[string]interface{})
|
||
|
||
err := json.Unmarshal([]byte(content), ¶m)
|
||
if err != nil {
|
||
fmt.Println("发送钉钉错误", err)
|
||
return
|
||
}
|
||
|
||
ok := notify.SendDingtalkToChan(dingtalk, param)
|
||
if !ok {
|
||
fmt.Println("发送钉钉错误", dingtalk)
|
||
}
|
||
} else if j.task.NotifyType == 3 && len(wechat) > 0 {
|
||
//微信
|
||
param := make(map[string]string)
|
||
err := json.Unmarshal([]byte(content), ¶m)
|
||
if err != nil {
|
||
fmt.Println("发送微信错误", err)
|
||
return
|
||
}
|
||
|
||
ok := notify.SendWechatToChan(phone, param)
|
||
if !ok {
|
||
fmt.Println("发送微信错误", phone)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
j.logId, _ = models.TaskLogAdd(log)
|
||
|
||
// 更新上次执行时间
|
||
j.task.PrevTime = t.Unix()
|
||
j.task.ExecuteTimes++
|
||
j.task.Update("PrevTime", "ExecuteTimes")
|
||
}
|
||
|
||
//冗余代码
|
||
type adminInfo struct {
|
||
Id int
|
||
Email string
|
||
Phone string
|
||
Dingtalk string
|
||
Wechat string
|
||
RealName string
|
||
}
|
||
|
||
func AllAdminInfo(adminIds string) []*adminInfo {
|
||
Filters := make([]interface{}, 0)
|
||
Filters = append(Filters, "status", 1)
|
||
//Filters = append(Filters, "id__gt", 1)
|
||
var notifyUserIds []int
|
||
if adminIds != "0" && adminIds != "" {
|
||
notifyUserIdsStr := strings.Split(adminIds, ",")
|
||
for _, v := range notifyUserIdsStr {
|
||
i, _ := strconv.Atoi(v)
|
||
notifyUserIds = append(notifyUserIds, i)
|
||
}
|
||
Filters = append(Filters, "id__in", notifyUserIds)
|
||
}
|
||
Result, _ := models.AdminGetList(1, 1000, Filters...)
|
||
|
||
adminInfos := make([]*adminInfo, 0)
|
||
for _, v := range Result {
|
||
ai := adminInfo{
|
||
Id: v.Id,
|
||
Email: v.Email,
|
||
Phone: v.Phone,
|
||
Dingtalk: v.Dingtalk,
|
||
Wechat: v.Wechat,
|
||
RealName: v.RealName,
|
||
}
|
||
adminInfos = append(adminInfos, &ai)
|
||
}
|
||
|
||
return adminInfos
|
||
}
|
||
|
||
func gbkAsUtf8(str string) string {
|
||
srcDecoder := mahonia.NewDecoder("gbk")
|
||
desDecoder := mahonia.NewDecoder("utf-8")
|
||
resStr := srcDecoder.ConvertString(str)
|
||
_, resBytes, _ := desDecoder.Translate([]byte(resStr), true)
|
||
return string(resBytes)
|
||
}
|
||
|
||
//任务识别码
|
||
func jobKey(taskId, serverId int) int {
|
||
return taskId*10000 + serverId
|
||
}
|