foundSeq

models/const.go

@@ -30,4 +30,5 @@ const (
 	OneHour                        = 1
 	OneMinutes                     = 1
 	TwoMinutes                     = 2
+	DefaultGpuSeq                  = 999
 )

operate/docker.go

@@ -8,6 +8,7 @@ import (
 	"example.com/m/db"
 	"example.com/m/log"
 	"example.com/m/models"
+	"example.com/m/utils"
 	"fmt"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -391,36 +392,54 @@ func (d *DockerOp) getContainerInfo(id string) (types.Container, error) {
 }
 
 func (d *DockerOp) checkGpuUsage(info *nodemanagerV2.HardwareInfo, modelInfo *models.ModelInfo, dockerCmd *models.DockerCmd) int32 {
-	envMap := make(map[string]string, 0)
-	var res int32 = 999
+	var res int32 = models.DefaultGpuSeq
+	// 匹配首次启动模型的硬件信息数据
 	gpu := info.GPU
 	isMatch := false
 	for _, gpuInfo := range gpu {
 		if gpuInfo.MemFree > modelInfo.RunningMem {
-			envMap[models.CudaEnv] = strconv.FormatInt(int64(gpuInfo.Seq), 10)
 			res = gpuInfo.Seq
 			isMatch = true
 			break
 		}
 	}
 	if !isMatch {
-		runningModel := db.GetRunningModel()
-		if len(runningModel) == 0 {
-			return 0
+		res = d.foundSeq(modelInfo)
+	}
+	return res
+}
+
+func (d *DockerOp) foundSeq(modelInfo *models.ModelInfo) int32 {
+	var res int32 = models.DefaultGpuSeq
+	isMatch := false
+	// 获取当前启动的模型
+	runningModel := d.ListContainer()
+	for _, containerInfo := range runningModel {
+		runningModelInfo, err := db.GetModel(containerInfo.Image)
+		if err != nil || runningModelInfo == nil {
+			continue
 		}
-		for _, modelInfo := range runningModel {
-			if modelInfo.RunningMem > modelInfo.RunningMem {
-				isMatch = true
-				d.StopContainer(modelInfo.ContainerId)
-				envMap[models.CudaEnv] = strconv.FormatInt(int64(modelInfo.GpuSeq), 10)
-				break
+		if runningModelInfo.RunningMem > modelInfo.RunningMem {
+			isMatch = true
+			d.StopContainer(containerInfo.ID)
+			if modelInfo.GpuSeq != models.DefaultGpuSeq {
+				res = modelInfo.GpuSeq
+			} else {
+				time.Sleep(time.Second * 2)
+				hardwareInfo := utils.GetHardwareInfo(conf.GetConfig().HardwareUrl)
+				for _, gpuInfo := range hardwareInfo.Data.Gpus {
+					if gpuInfo.MemFree > modelInfo.RunningMem {
+						res = gpuInfo.Seq
+						isMatch = true
+						break
+					}
+				}
 			}
+			break
 		}
 	}
-	if isMatch {
-		//nm.ModelRunningBeforeMem[modelInfo.ImageName] = dockerCmd.RunningBeforeMem
-		gpuSeq, _ := strconv.ParseInt(dockerCmd.EnvMap[models.CudaEnv], 10, 32)
-		res = int32(gpuSeq)
+	if !isMatch {
+		d.foundSeq(modelInfo)
 	}
 	return res
 }