mirror of
https://github.com/YFGaia/dify-plus.git
synced 2026-06-04 10:14:00 +08:00
fix: Dify 1.8.1问题修复
本次提交整合了多个功能改进和问题修复: 主要功能: - 批量工作流处理功能完善,支持 Excel 上传和进度跟踪 - 管理中心反向代理和转发配置优化 - 用户同步添加互斥锁,防止并发问题 - 计费系统和额度显示优化 - AI 绘图功能扩展 前端改进: - 文本生成应用显示修复 - 批量任务进度展示优化 - 按钮样式和 CSS 优化,禁止换行 - 多语言支持完善(新增印尼语等) - 构建镜像逻辑优化 - 批量处理进度管理器实现 后端改进: - Docker Compose 配置升级 - 队列任务和 Worker Pool 优化 - Admin API 初始化和验证逻辑改进 - 数据库迁移和初始化完善 - 静态变量处理优化 - URL 签名助手实现 - Celery 扩展优化 - 代码和导入包问题修复(idea 自动调整代码位置) 技术改进: - 兼容性修复 (flask-restx, jschardet) - 钉钉 Web API 版本更新 - 代码格式化和导入包问题修复 - 日志处理优化 - 工作流循环管理优化 Docker 相关: - Nginx 配置更新 - 容器启动脚本优化 - 镜像构建流程改进 - docker-compose.dify-plus.yaml 大幅更新 管理后台: - 工作流批量处理 API 实现 - 工作池初始化 - 批量工作流服务实现 - 转发扩展配置 - 用户服务扩展
This commit is contained in:
@@ -9,6 +9,7 @@ type ApiGroup struct {
|
||||
SystemApi
|
||||
TestApi
|
||||
SystemOAuth2Api
|
||||
BatchWorkflowApi
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -0,0 +1,657 @@
|
||||
package gaia
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/global"
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/model/gaia"
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/model/gaia/request"
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/utils"
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
"golang.org/x/text/transform"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/model/common/response"
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/service"
|
||||
gaiaService "github.com/flipped-aurora/gin-vue-admin/server/service/gaia"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type BatchWorkflowApi struct{}
|
||||
|
||||
var batchWorkflowService = service.ServiceGroupApp.GaiaServiceGroup.BatchWorkflowService
|
||||
|
||||
// CreateBatchWorkflow 创建批量处理工作流
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 创建批量处理工作流
|
||||
// @Description 上传CSV文件并创建批量处理工作流
|
||||
// @Accept multipart/form-data
|
||||
// @Produce application/json
|
||||
// @Param file formData file true "CSV文件"
|
||||
// @Param installed_id formData string true "安装的应用ID"
|
||||
// @Param app_id formData string true "应用ID"
|
||||
// @Param tenant_id formData string true "租户ID"
|
||||
// @Success 200 {object} response.Response{data=gaia.BatchWorkflow} "成功"
|
||||
// @Router /gaia/workflow/batch/processing [post]
|
||||
func (api *BatchWorkflowApi) CreateBatchWorkflow(c *gin.Context) {
|
||||
// 获取表单参数
|
||||
userID := utils.GetUserID(c)
|
||||
installedID := c.PostForm("installed_id")
|
||||
keyNameMappingStr := c.PostForm("key_name_mapping")
|
||||
|
||||
if installedID == "" {
|
||||
response.FailWithMessage("缺少必要参数", c)
|
||||
return
|
||||
}
|
||||
|
||||
// 解析key-name映射
|
||||
var keyNameMapping map[string]string
|
||||
if keyNameMappingStr != "" {
|
||||
if err := json.Unmarshal([]byte(keyNameMappingStr), &keyNameMapping); err != nil {
|
||||
response.FailWithMessage("解析key_name_mapping失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 获取上传的文件
|
||||
file, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
response.FailWithMessage("获取文件失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
// 打开上传的文件
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
response.FailWithMessage("打开文件失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
// 读取文件内容并检测编码
|
||||
content, err := io.ReadAll(src)
|
||||
if err != nil {
|
||||
response.FailWithMessage("读取文件内容失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
// 尝试不同编码解析CSV
|
||||
var data [][]string
|
||||
var parseErr error
|
||||
|
||||
// 1. 先尝试UTF-8读取,使用宽松的CSV解析器配置
|
||||
reader := bytes.NewReader(content)
|
||||
csvReader := csv.NewReader(reader)
|
||||
csvReader.LazyQuotes = true // 允许懒惰引号
|
||||
csvReader.TrimLeadingSpace = true // 去除前导空格
|
||||
data, parseErr = csvReader.ReadAll()
|
||||
|
||||
// 2. 如果UTF-8失败或包含乱码,尝试GBK编码
|
||||
if parseErr != nil || containsGarbledText(data) {
|
||||
decoder := simplifiedchinese.GBK.NewDecoder()
|
||||
gbkReader := transform.NewReader(bytes.NewReader(content), decoder)
|
||||
|
||||
csvReader = csv.NewReader(gbkReader)
|
||||
csvReader.LazyQuotes = true // 允许懒惰引号
|
||||
csvReader.TrimLeadingSpace = true // 去除前导空格
|
||||
data, parseErr = csvReader.ReadAll()
|
||||
|
||||
// 3. 如果GBK也失败,尝试GB18030编码
|
||||
if parseErr != nil || containsGarbledText(data) {
|
||||
gb18030Decoder := simplifiedchinese.GB18030.NewDecoder()
|
||||
gb18030Reader := transform.NewReader(bytes.NewReader(content), gb18030Decoder)
|
||||
|
||||
csvReader = csv.NewReader(gb18030Reader)
|
||||
csvReader.LazyQuotes = true // 允许懒惰引号
|
||||
csvReader.TrimLeadingSpace = true // 去除前导空格
|
||||
data, parseErr = csvReader.ReadAll()
|
||||
}
|
||||
}
|
||||
|
||||
// 4. 如果以上方法都失败,尝试最后的兜底解析方法
|
||||
if parseErr != nil {
|
||||
data, parseErr = parseCSVWithFallback(content)
|
||||
}
|
||||
|
||||
if parseErr != nil {
|
||||
response.FailWithMessage("解析CSV文件失败,请检查文件格式。错误详情: "+parseErr.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
// 创建批量处理工作流
|
||||
batchWorkflow, err := batchWorkflowService.CreateBatchWorkflow(
|
||||
userID, installedID, file.Filename, data, keyNameMapping)
|
||||
if err != nil {
|
||||
// 特别处理数据库连接问题
|
||||
if strings.Contains(err.Error(), "数据库连接未初始化") {
|
||||
response.FailWithMessage("系统初始化中,请稍后重试", c)
|
||||
} else {
|
||||
response.FailWithMessage("创建批量处理失败: "+err.Error(), c)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithData(batchWorkflow, c)
|
||||
}
|
||||
|
||||
// GetBatchWorkflow 获取批量处理信息
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 获取批量处理信息
|
||||
// @Description 根据ID获取批量处理信息
|
||||
// @Produce application/json
|
||||
// @Param id path string true "批量处理ID"
|
||||
// @Success 200 {object} response.Response{data=gaia.BatchWorkflow} "成功"
|
||||
// @Router /gaia/workflow/batch/{id} [get]
|
||||
func (api *BatchWorkflowApi) GetBatchWorkflow(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
if id == "" {
|
||||
response.FailWithMessage("缺少批量处理ID", c)
|
||||
return
|
||||
}
|
||||
|
||||
batchWorkflow, err := batchWorkflowService.GetBatchWorkflow(id)
|
||||
if err != nil {
|
||||
response.FailWithMessage("获取批量处理信息失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithData(batchWorkflow, c)
|
||||
}
|
||||
|
||||
// GetBatchWorkflowTasks 获取批量处理任务列表
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 获取批量处理任务列表
|
||||
// @Description 根据批量处理ID获取任务列表
|
||||
// @Produce application/json
|
||||
// @Param id path string true "批量处理ID"
|
||||
// @Success 200 {object} response.Response{data=[]gaia.BatchWorkflowTask} "成功"
|
||||
// @Router /gaia/workflow/batch/{id}/tasks [get]
|
||||
func (api *BatchWorkflowApi) GetBatchWorkflowTasks(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
if id == "" {
|
||||
response.FailWithMessage("缺少批量处理ID", c)
|
||||
return
|
||||
}
|
||||
|
||||
tasks, err := batchWorkflowService.GetBatchWorkflowTasks(id)
|
||||
if err != nil {
|
||||
response.FailWithMessage("获取任务列表失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithData(tasks, c)
|
||||
}
|
||||
|
||||
// GetBatchWorkflowProgress 获取批量处理进度
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 获取批量处理进度
|
||||
// @Description 根据ID获取批量处理进度信息
|
||||
// @Produce application/json
|
||||
// @Param id path string true "批量处理ID"
|
||||
// @Success 200 {object} response.Response{data=map[string]interface{}} "成功"
|
||||
// @Router /gaia/workflow/batch/{id}/progress [get]
|
||||
func (api *BatchWorkflowApi) GetBatchWorkflowProgress(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
if id == "" {
|
||||
response.FailWithMessage("缺少批量处理ID", c)
|
||||
return
|
||||
}
|
||||
|
||||
progress, err := batchWorkflowService.GetBatchWorkflowProgress(id)
|
||||
if err != nil {
|
||||
response.FailWithMessage("获取进度信息失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithData(progress, c)
|
||||
}
|
||||
|
||||
// StopBatchWorkflow 停止批量处理
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 停止批量处理
|
||||
// @Description 根据ID停止批量处理
|
||||
// @Produce application/json
|
||||
// @Param id path string true "批量处理ID"
|
||||
// @Success 200 {object} response.Response "成功"
|
||||
// @Router /gaia/workflow/batch/{id}/stop [post]
|
||||
func (api *BatchWorkflowApi) StopBatchWorkflow(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
if id == "" {
|
||||
response.FailWithMessage("缺少批量处理ID", c)
|
||||
return
|
||||
}
|
||||
|
||||
err := batchWorkflowService.StopBatchWorkflow(id)
|
||||
if err != nil {
|
||||
response.FailWithMessage("停止批量处理失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithMessage("停止成功", c)
|
||||
}
|
||||
|
||||
// RetryBatchWorkflow 重试批量处理(重新开始所有任务)
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 重试批量处理
|
||||
// @Description 根据ID重试批量处理,重置所有任务从头开始
|
||||
// @Produce application/json
|
||||
// @Param id path string true "批量处理ID"
|
||||
// @Success 200 {object} response.Response "成功"
|
||||
// @Router /gaia/workflow/batch/{id}/retry [post]
|
||||
func (api *BatchWorkflowApi) RetryBatchWorkflow(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
if id == "" {
|
||||
response.FailWithMessage("缺少批量处理ID", c)
|
||||
return
|
||||
}
|
||||
|
||||
err := batchWorkflowService.RetryBatchWorkflow(id)
|
||||
if err != nil {
|
||||
response.FailWithMessage("重试批量处理失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithMessage("重试成功,所有任务已重置", c)
|
||||
}
|
||||
|
||||
// RetryFailedTasks 仅重试失败的任务
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 仅重试失败的任务
|
||||
// @Description 根据ID仅重试失败的任务,保留已完成的任务
|
||||
// @Produce application/json
|
||||
// @Param id path string true "批量处理ID"
|
||||
// @Success 200 {object} response.Response "成功"
|
||||
// @Router /gaia/workflow/batch/{id}/retry-failed [post]
|
||||
func (api *BatchWorkflowApi) RetryFailedTasks(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
if id == "" {
|
||||
response.FailWithMessage("缺少批量处理ID", c)
|
||||
return
|
||||
}
|
||||
|
||||
err := batchWorkflowService.RetryFailedTasks(id)
|
||||
if err != nil {
|
||||
response.FailWithMessage("重试失败任务失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithMessage("失败任务重试成功", c)
|
||||
}
|
||||
|
||||
// ResumeBatchWorkflow 恢复批量处理
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 恢复批量处理
|
||||
// @Description 根据ID恢复批量处理
|
||||
// @Produce application/json
|
||||
// @Param id path string true "批量处理ID"
|
||||
// @Success 200 {object} response.Response "成功"
|
||||
// @Router /gaia/workflow/batch/{id}/resume [post]
|
||||
func (api *BatchWorkflowApi) ResumeBatchWorkflow(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
if id == "" {
|
||||
response.FailWithMessage("缺少批量处理ID", c)
|
||||
return
|
||||
}
|
||||
|
||||
err := batchWorkflowService.ResumeBatchWorkflow(id)
|
||||
if err != nil {
|
||||
response.FailWithMessage("恢复批量处理失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithMessage("恢复成功", c)
|
||||
}
|
||||
|
||||
// ResetBatchWorkflowErrorCount 重置批量工作流错误计数
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 重置批量工作流错误计数
|
||||
// @Description 重置指定批量工作流的错误计数,恢复用户并发位
|
||||
// @Produce application/json
|
||||
// @Param id path string true "批量处理ID"
|
||||
// @Success 200 {object} response.Response "成功"
|
||||
// @Router /gaia/workflow/batch/{id}/reset-error-count [post]
|
||||
func (api *BatchWorkflowApi) ResetBatchWorkflowErrorCount(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
if id == "" {
|
||||
response.FailWithMessage("缺少批量处理ID", c)
|
||||
return
|
||||
}
|
||||
|
||||
// 调用worker_pool中的重置函数
|
||||
err := gaiaService.ResetBatchWorkflowErrorCount(id)
|
||||
if err != nil {
|
||||
response.FailWithMessage("重置错误计数失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithMessage("错误计数已重置,用户并发位将恢复", c)
|
||||
}
|
||||
|
||||
// ResetUserErrorCount 重置用户所有批量工作流错误计数
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 重置用户所有批量工作流错误计数
|
||||
// @Description 重置指定用户所有批量工作流的错误计数,恢复用户并发位
|
||||
// @Produce application/json
|
||||
// @Success 200 {object} response.Response "成功"
|
||||
// @Router /gaia/workflow/batch/reset-user-error-count [post]
|
||||
func (api *BatchWorkflowApi) ResetUserErrorCount(c *gin.Context) {
|
||||
userID := utils.GetUserID(c)
|
||||
|
||||
// 调用worker_pool中的重置函数
|
||||
err := gaiaService.ResetUserErrorCount(userID)
|
||||
if err != nil {
|
||||
response.FailWithMessage("重置用户错误计数失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithMessage("用户所有批量工作流错误计数已重置,并发位将恢复", c)
|
||||
}
|
||||
|
||||
// DownloadBatchWorkflowResults 下载批量处理结果
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 下载批量处理结果
|
||||
// @Description 根据ID下载批量处理结果
|
||||
// @Produce text/csv
|
||||
// @Param id path string true "批量处理ID"
|
||||
// @Success 200 {file} file "CSV文件"
|
||||
// @Router /gaia/workflow/batch/{id}/download [get]
|
||||
func (api *BatchWorkflowApi) DownloadBatchWorkflowResults(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
if id == "" {
|
||||
response.FailWithMessage("缺少批量处理ID", c)
|
||||
return
|
||||
}
|
||||
|
||||
// 获取批量处理信息
|
||||
flow, err := batchWorkflowService.GetBatchWorkflow(id)
|
||||
if err != nil {
|
||||
response.FailWithMessage("获取批量处理信息失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
// 获取任务列表
|
||||
tasks, err := batchWorkflowService.GetBatchWorkflowTasks(id)
|
||||
if err != nil {
|
||||
response.FailWithMessage("获取任务列表失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
// 生成CSV内容
|
||||
csvContent := generateCSVFromTasks(flow, tasks)
|
||||
csvBytes := []byte(csvContent)
|
||||
|
||||
// 添加 UTF-8 BOM 以确保在 Excel 中正确显示中文
|
||||
bom := []byte{0xEF, 0xBB, 0xBF}
|
||||
fullContent := append(bom, csvBytes...)
|
||||
|
||||
// 设置响应头
|
||||
filename := fmt.Sprintf("batch_results_%s.csv", id)
|
||||
c.Header("Content-Type", "text/csv; charset=utf-8")
|
||||
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename*=UTF-8''%s", filename))
|
||||
c.Header("Content-Length", fmt.Sprintf("%d", len(fullContent)))
|
||||
c.Header("Cache-Control", "no-cache, no-store, must-revalidate")
|
||||
c.Header("Pragma", "no-cache")
|
||||
c.Header("Expires", "0")
|
||||
|
||||
c.Data(http.StatusOK, "text/csv; charset=utf-8", fullContent)
|
||||
}
|
||||
|
||||
// parseCSVWithFallback 兜底的CSV解析方法,用于处理格式不规范的CSV文件
|
||||
func parseCSVWithFallback(content []byte) ([][]string, error) {
|
||||
// 将内容转换为字符串并按行分割
|
||||
contentStr := string(content)
|
||||
lines := strings.Split(contentStr, "\n")
|
||||
|
||||
var result [][]string
|
||||
for i, line := range lines {
|
||||
// 跳过空行
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// 尝试简单的逗号分割
|
||||
fields := strings.Split(line, ",")
|
||||
|
||||
// 清理字段:去除多余的引号和空格
|
||||
for j, field := range fields {
|
||||
field = strings.TrimSpace(field)
|
||||
// 如果字段被引号包围,去除引号
|
||||
if len(field) >= 2 && field[0] == '"' && field[len(field)-1] == '"' {
|
||||
field = field[1 : len(field)-1]
|
||||
// 处理转义的引号
|
||||
field = strings.ReplaceAll(field, `""`, `"`)
|
||||
}
|
||||
fields[j] = field
|
||||
}
|
||||
|
||||
result = append(result, fields)
|
||||
|
||||
// 如果解析失败超过100行,停止解析
|
||||
if i > 100 && len(result) == 0 {
|
||||
return nil, fmt.Errorf("无法解析CSV文件:格式不正确")
|
||||
}
|
||||
}
|
||||
|
||||
if len(result) == 0 {
|
||||
return nil, fmt.Errorf("CSV文件为空或格式无法识别")
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// containsGarbledText 检测是否包含乱码文本
|
||||
func containsGarbledText(data [][]string) bool {
|
||||
// 检查前几行是否包含类似乱码的字符
|
||||
checkRows := 3
|
||||
if len(data) < checkRows {
|
||||
checkRows = len(data)
|
||||
}
|
||||
|
||||
for i := 0; i < checkRows; i++ {
|
||||
for _, cell := range data[i] {
|
||||
// 检查是否包含典型的编码错误字符
|
||||
for _, char := range cell {
|
||||
// 检查是否为替换字符(U+FFFD)或其他异常字符
|
||||
if char == '�' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// 检查特定的GBK乱码模式
|
||||
if strings.Contains(cell, "��") || strings.Contains(cell, "Ŀ") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// generateCSVFromTasks 从任务生成CSV内容
|
||||
func generateCSVFromTasks(flow *gaia.BatchWorkflow, tasks []gaia.BatchWorkflowTask) string {
|
||||
if len(tasks) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 解析第一个任务的输入参数来获取列名
|
||||
var firstTaskInputs map[string]string
|
||||
if err := json.Unmarshal([]byte(tasks[0].Inputs), &firstTaskInputs); err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
w := csv.NewWriter(buf)
|
||||
|
||||
// 标题:输入列 + 处理结果 + 状态
|
||||
var nameList []string
|
||||
var keyMap map[string]string
|
||||
_ = json.Unmarshal([]byte(flow.KeyName), &keyMap)
|
||||
headers := make([]string, 0, len(keyMap))
|
||||
for key, value := range keyMap {
|
||||
headers = append(headers, key)
|
||||
nameList = append(nameList, value)
|
||||
}
|
||||
headers = append(headers, "生成结果")
|
||||
_ = w.Write(headers)
|
||||
|
||||
// 行数据
|
||||
for _, task := range tasks {
|
||||
var inputs map[string]string
|
||||
if err := json.Unmarshal([]byte(task.Inputs), &inputs); err != nil {
|
||||
continue
|
||||
}
|
||||
var text string
|
||||
row := make([]string, 0, len(headers))
|
||||
var result request.WorkflowBatchProcessing
|
||||
for _, value := range nameList {
|
||||
row = append(row, inputs[value])
|
||||
}
|
||||
if err := json.Unmarshal([]byte(task.Result), &result); err == nil {
|
||||
for key, v := range result.Outputs {
|
||||
if key == "task_id" {
|
||||
continue
|
||||
}
|
||||
text += fmt.Sprintf("%s\r", v)
|
||||
}
|
||||
}
|
||||
row = append(row, text)
|
||||
_ = w.Write(row)
|
||||
}
|
||||
|
||||
w.Flush()
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// GetWorkerPoolStatus 获取工作池状态
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 获取工作池状态
|
||||
// @Description 获取当前工作池的运行状态和统计信息
|
||||
// @Accept application/json
|
||||
// @Produce application/json
|
||||
// @Success 200 {object} response.Response{data=map[string]interface{}} "成功"
|
||||
// @Router /gaia/workflow/worker-pool/status [get]
|
||||
func (api *BatchWorkflowApi) GetWorkerPoolStatus(c *gin.Context) {
|
||||
pool := batchWorkflowService.GetWorkerPool()
|
||||
if pool == nil {
|
||||
response.FailWithMessage("工作池未初始化", c)
|
||||
return
|
||||
}
|
||||
|
||||
status := pool.GetStatus()
|
||||
response.OkWithData(status, c)
|
||||
}
|
||||
|
||||
// RestartWorkerPool 重启工作池
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 重启工作池
|
||||
// @Description 停止当前工作池并重新启动
|
||||
// @Accept application/json
|
||||
// @Produce application/json
|
||||
// @Param workers query int false "worker数量" default(5)
|
||||
// @Success 200 {object} response.Response "成功"
|
||||
// @Router /gaia/workflow/worker-pool/restart [post]
|
||||
func (api *BatchWorkflowApi) RestartWorkerPool(c *gin.Context) {
|
||||
workers := global.GVA_CONFIG.System.WorkFlowNumber
|
||||
if workersParam := c.Query("workers"); workersParam != "" {
|
||||
if w, err := strconv.Atoi(workersParam); err == nil && w > 0 && w <= 20 {
|
||||
workers = w
|
||||
}
|
||||
}
|
||||
|
||||
// 停止当前工作池
|
||||
batchWorkflowService.StopWorkerPool()
|
||||
|
||||
// 启动新的工作池
|
||||
batchWorkflowService.InitWorkerPool(workers)
|
||||
|
||||
response.OkWithMessage("工作池重启成功", c)
|
||||
}
|
||||
|
||||
// StopWorkerPool 停止工作池
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 停止工作池
|
||||
// @Description 停止当前工作池
|
||||
// @Accept application/json
|
||||
// @Produce application/json
|
||||
// @Success 200 {object} response.Response "成功"
|
||||
// @Router /gaia/workflow/worker-pool/stop [post]
|
||||
func (api *BatchWorkflowApi) StopWorkerPool(c *gin.Context) {
|
||||
batchWorkflowService.StopWorkerPool()
|
||||
response.OkWithMessage("工作池已停止", c)
|
||||
}
|
||||
|
||||
// StartWorkerPool 启动工作池
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 启动工作池
|
||||
// @Description 启动工作池
|
||||
// @Accept application/json
|
||||
// @Produce application/json
|
||||
// @Param workers query int false "worker数量" default(5)
|
||||
// @Success 200 {object} response.Response "成功"
|
||||
// @Router /gaia/workflow/worker-pool/start [post]
|
||||
func (api *BatchWorkflowApi) StartWorkerPool(c *gin.Context) {
|
||||
workers := global.GVA_CONFIG.System.WorkFlowNumber
|
||||
if workersParam := c.Query("workers"); workersParam != "" {
|
||||
if w, err := strconv.Atoi(workersParam); err == nil && w > 0 && w <= 20 {
|
||||
workers = w
|
||||
}
|
||||
}
|
||||
|
||||
batchWorkflowService.InitWorkerPool(workers)
|
||||
response.OkWithMessage("工作池启动成功", c)
|
||||
}
|
||||
|
||||
// GetBatchWorkflowList 获取最近30天的批量工作流列表
|
||||
// @Tags BatchWorkflow
|
||||
// @Summary 获取最近30天的批量工作流列表
|
||||
// @Description 获取指定用户最近30天的批量工作流列表,支持分页和按应用过滤
|
||||
// @Accept application/json
|
||||
// @Produce application/json
|
||||
// @Param installed_id query string false "安装的应用ID"
|
||||
// @Param page query int false "页码" default(1)
|
||||
// @Param limit query int false "每页数量" default(10)
|
||||
// @Success 200 {object} response.Response{data=map[string]interface{}} "成功"
|
||||
// @Router /gaia/workflow/batch/list [get]
|
||||
func (api *BatchWorkflowApi) GetBatchWorkflowList(c *gin.Context) {
|
||||
userID := utils.GetUserID(c)
|
||||
installedID := c.Query("installed_id")
|
||||
|
||||
// 解析分页参数
|
||||
page := 1
|
||||
limit := 10
|
||||
|
||||
if pageParam := c.Query("page"); pageParam != "" {
|
||||
if p, err := strconv.Atoi(pageParam); err == nil && p > 0 {
|
||||
page = p
|
||||
}
|
||||
}
|
||||
|
||||
if limitParam := c.Query("limit"); limitParam != "" {
|
||||
if l, err := strconv.Atoi(limitParam); err == nil && l > 0 && l <= 100 {
|
||||
limit = l
|
||||
}
|
||||
}
|
||||
|
||||
// 调用服务层方法
|
||||
batchWorkflows, total, err := batchWorkflowService.GetBatchWorkflowList(userID, installedID, page, limit)
|
||||
if err != nil {
|
||||
response.FailWithMessage("获取批量工作流列表失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
// 计算分页信息
|
||||
totalPages := (total + int64(limit) - 1) / int64(limit)
|
||||
hasMore := int64(page) < totalPages
|
||||
|
||||
response.OkWithData(map[string]interface{}{
|
||||
"items": batchWorkflows,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"limit": limit,
|
||||
"total_pages": totalPages,
|
||||
"has_more": hasMore,
|
||||
}, c)
|
||||
}
|
||||
@@ -160,6 +160,7 @@ system:
|
||||
use-strict-auth: false
|
||||
user_default-group-id: "888"
|
||||
docker-run: true
|
||||
work_flow_number: 100
|
||||
tencent-cos:
|
||||
bucket: xxxxx-10005608
|
||||
region: ap-shanghai
|
||||
|
||||
@@ -10,7 +10,7 @@ captcha:
|
||||
open-captcha: 0
|
||||
open-captcha-timeout: 3600
|
||||
jwt:
|
||||
signing-key:
|
||||
signing-key: sk-9f73s3ljTXVcMT3Blb3ljTqtsKiGHXVcMT3BlbkFJLK7U
|
||||
expires-time: 1d
|
||||
buffer-time: 1d
|
||||
issuer: CLOUD
|
||||
@@ -61,6 +61,7 @@ system:
|
||||
use-mongo: false
|
||||
use-strict-auth: false
|
||||
user_default-group-id: "888"
|
||||
work_flow_number: 100
|
||||
zap:
|
||||
level: info
|
||||
prefix: '[gaia/server]'
|
||||
|
||||
@@ -12,6 +12,7 @@ type System struct {
|
||||
UseMongo bool `mapstructure:"use-mongo" json:"use-mongo" yaml:"use-mongo"` // 使用mongo
|
||||
UseStrictAuth bool `mapstructure:"use-strict-auth" json:"use-strict-auth" yaml:"use-strict-auth"` // 使用树形角色分配模式
|
||||
// Extend: Start Custom Configuration
|
||||
WorkFlowNumber int `mapstructure:"work_flow_number" default:"200" json:"work_flow_number" yaml:"work_flow_number"`
|
||||
UserDefaultGroupID string `mapstructure:"user_default-group-id" default:"888" json:"user_default-group-id" yaml:"user_default-group-id"` // 用户默认群组id
|
||||
DockerRun bool `mapstructure:"docker-run" default:false json:"docker-run" yaml:"docker-run"` // 是否在docker中运行,如果是的话,无需自动生成jwtkey,直接填sk-9f73s3ljTXVcMT3Blb3ljTqtsKiGHXVcMT3BlbkFJLK7U与dify保持一致
|
||||
// Extend: Stop Custom Configuration
|
||||
|
||||
@@ -14,6 +14,19 @@ import (
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/global"
|
||||
)
|
||||
|
||||
// Extend: Override JWT signing key from environment variable
|
||||
// This ensures admin-server uses the same JWT signing key as the API server
|
||||
func overrideJWTSigningKeyFromEnv() {
|
||||
// Check JWT_SIGNING_KEY first, then fall back to SECRET_KEY
|
||||
if jwtKey := os.Getenv("JWT_SIGNING_KEY"); jwtKey != "" {
|
||||
global.GVA_CONFIG.JWT.SigningKey = jwtKey
|
||||
fmt.Printf("JWT signing key overridden from JWT_SIGNING_KEY environment variable\n")
|
||||
} else if secretKey := os.Getenv("SECRET_KEY"); secretKey != "" {
|
||||
global.GVA_CONFIG.JWT.SigningKey = secretKey
|
||||
fmt.Printf("JWT signing key overridden from SECRET_KEY environment variable\n")
|
||||
}
|
||||
}
|
||||
|
||||
// Viper //
|
||||
// 优先级: 命令行 > 环境变量 > 默认值
|
||||
// Author [SliverHorn](https://github.com/SliverHorn)
|
||||
@@ -60,11 +73,16 @@ func Viper(path ...string) *viper.Viper {
|
||||
if err = v.Unmarshal(&global.GVA_CONFIG); err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
// Extend: Override JWT signing key from environment variable
|
||||
overrideJWTSigningKeyFromEnv()
|
||||
})
|
||||
if err = v.Unmarshal(&global.GVA_CONFIG); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Extend: Override JWT signing key from environment variable after initial load
|
||||
overrideJWTSigningKeyFromEnv()
|
||||
|
||||
// root 适配性 根据root位置去找到对应迁移位置,保证root路径有效
|
||||
global.GVA_CONFIG.AutoCode.Root, _ = filepath.Abs("..")
|
||||
|
||||
|
||||
@@ -66,6 +66,9 @@ func (e *ensureTables) MigrateTable(ctx context.Context) (context.Context, error
|
||||
gaia.AppRequestTestBatch{},
|
||||
gaia.AppRequestTest{},
|
||||
gaia.SystemIntegration{}, // Extend System Integration
|
||||
gaia.ForwardingExtend{}, // Extend Forwarding Extend
|
||||
gaia.BatchWorkflow{}, // Extend Batch Workflow
|
||||
gaia.BatchWorkflowTask{}, // Extend Batch Workflow Task
|
||||
sysModel.SysUserGlobalCode{}, // Extend Global Code
|
||||
// Extend gaia model
|
||||
}
|
||||
@@ -112,6 +115,9 @@ func (e *ensureTables) TableCreated(ctx context.Context) bool {
|
||||
gaia.AppRequestTestBatch{},
|
||||
gaia.AppRequestTest{},
|
||||
gaia.SystemIntegration{}, // Extend System Integration
|
||||
gaia.ForwardingExtend{}, // Extend Forwarding Extend
|
||||
gaia.BatchWorkflow{}, // Extend Batch Workflow
|
||||
gaia.BatchWorkflowTask{}, // Extend Batch Workflow Task
|
||||
sysModel.SysUserGlobalCode{}, // Extend Global Code
|
||||
// Extend gaia model
|
||||
}
|
||||
|
||||
@@ -38,7 +38,6 @@ func Gorm() *gorm.DB {
|
||||
func RegisterTables() {
|
||||
db := global.GVA_DB
|
||||
err := db.AutoMigrate(
|
||||
|
||||
system.SysApi{},
|
||||
system.SysIgnoreApi{},
|
||||
system.SysUser{},
|
||||
@@ -68,6 +67,9 @@ func RegisterTables() {
|
||||
gaia.AppRequestTestBatch{},
|
||||
gaia.AppRequestTest{},
|
||||
gaia.SystemIntegration{}, // Extend System Integration
|
||||
gaia.ForwardingExtend{}, // Extend Forwarding Extend
|
||||
gaia.BatchWorkflow{}, // Extend Batch Workflow
|
||||
gaia.BatchWorkflowTask{}, // Extend Batch Workflow Task
|
||||
system.SysUserGlobalCode{}, // Extend Global Code
|
||||
// Extend gaia model
|
||||
)
|
||||
|
||||
@@ -20,5 +20,6 @@ func initBizRouter(routers ...*gin.RouterGroup) {
|
||||
gaiaRouter.InitTenantsRouter(privateGroup, publicGroup)
|
||||
gaiaRouter.InitTestRouter(privateGroup, publicGroup)
|
||||
gaiaRouter.InitSystemRouter(privateGroup)
|
||||
gaiaRouter.InitWorkflowRouter(privateGroup)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package initialize
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/global"
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/service/gaia"
|
||||
)
|
||||
|
||||
// InitWorkerPool 初始化工作池
|
||||
func InitWorkerPool() {
|
||||
// 从配置中获取worker数量,默认为5
|
||||
workerCount := 5
|
||||
if global.GVA_CONFIG.System.WorkFlowNumber > 0 {
|
||||
workerCount = global.GVA_CONFIG.System.WorkFlowNumber
|
||||
}
|
||||
global.GVA_LOG.Info(fmt.Sprintf("正在启动批量任务工作池,工作器数量: %d", workerCount))
|
||||
gaia.InitWorkerPool(workerCount)
|
||||
global.GVA_LOG.Info("批量任务工作池启动完成")
|
||||
}
|
||||
|
||||
// StopWorkerPool 停止工作池(优雅关闭时调用)
|
||||
func StopWorkerPool() {
|
||||
global.GVA_LOG.Info("正在停止批量任务工作池...")
|
||||
gaia.StopWorkerPool()
|
||||
global.GVA_LOG.Info("批量任务工作池已停止")
|
||||
}
|
||||
@@ -30,6 +30,7 @@ func main() {
|
||||
initialize.DBList()
|
||||
if global.GVA_DB != nil {
|
||||
initialize.RegisterTables() // 初始化表
|
||||
initialize.InitWorkerPool() // 初始化工作池
|
||||
// 程序结束前关闭数据库链接
|
||||
db, _ := global.GVA_DB.DB()
|
||||
defer db.Close()
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package gaia
|
||||
|
||||
import "time"
|
||||
|
||||
// 批量工作流状态常量
|
||||
const (
|
||||
BatchWorkflowStatusPending = "pending" // 待处理
|
||||
BatchWorkflowStatusProcessing = "processing" // 处理中
|
||||
BatchWorkflowStatusCompleted = "completed" // 已完成
|
||||
BatchWorkflowStatusFailed = "failed" // 失败
|
||||
BatchWorkflowStatusStopped = "stopped" // 已停止
|
||||
)
|
||||
|
||||
// 批量工作流任务状态常量
|
||||
const (
|
||||
BatchTaskStatusPending = "pending" // 待处理
|
||||
BatchTaskStatusQueued = "queued" // 队列中
|
||||
BatchTaskStatusRunning = "running" // 运行中
|
||||
BatchTaskStatusCompleted = "completed" // 已完成
|
||||
BatchTaskStatusFailed = "failed" // 失败
|
||||
BatchTaskStatusCancelled = "cancelled" // 已取消
|
||||
)
|
||||
|
||||
// 批量工作流错误消息常量
|
||||
const (
|
||||
ErrorInsufficientBalance = "余额不足,调用失败!"
|
||||
ErrorMaxRetryExceeded = "重试超过3次"
|
||||
ErrorWorkflowFailed = "工作流执行失败"
|
||||
ErrorCallAPIFailed = "调用Dify API失败"
|
||||
ErrorParseResultFailed = "解析API返回结果失败"
|
||||
)
|
||||
|
||||
// 批量工作流配置常量
|
||||
const (
|
||||
MaxTaskRetryCount = 3 // 最大任务重试次数
|
||||
ErrorPenaltyThreshold = 50 // 错误惩罚阈值(每50个错误减少1个并发位)
|
||||
)
|
||||
|
||||
// BatchWorkflow 批量工作流处理
|
||||
type BatchWorkflow struct {
|
||||
ID string `json:"id" gorm:"primaryKey;comment:批量处理ID"`
|
||||
UserID uint `json:"user_id" gorm:"index;comment:用户id"`
|
||||
InstalledID string `json:"installed_id" gorm:"not null;comment:安装的应用ID"`
|
||||
FileName string `json:"file_name" gorm:"not null;comment:上传的文件名"`
|
||||
TotalRows int `json:"total_rows" gorm:"not null;default:0;comment:总行数"`
|
||||
ProcessedRows int `json:"processed_rows" gorm:"not null;default:0;comment:已处理行数"`
|
||||
Status string `json:"status" gorm:"not null;default:'pending';comment:状态: pending, processing, completed, failed, stopped"`
|
||||
Results string `json:"results" gorm:"type:text;comment:处理结果"`
|
||||
KeyName string `json:"key_name" gorm:"type:text;comment:键名"`
|
||||
Error string `json:"error" gorm:"comment:错误信息"`
|
||||
ErrorCount int `json:"error_count" gorm:"not null;default:0;comment:累计错误次数"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"not null;default:CURRENT_TIMESTAMP(0);comment:创建时间"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"not null;default:CURRENT_TIMESTAMP(0);comment:更新时间"`
|
||||
}
|
||||
|
||||
// BatchWorkflowTask 批量工作流任务
|
||||
type BatchWorkflowTask struct {
|
||||
ID string `json:"id" gorm:"primaryKey;comment:任务ID"`
|
||||
BatchWorkflowID string `json:"batch_workflow_id" gorm:"not null;comment:批量处理ID"`
|
||||
RowIndex int `json:"row_index" gorm:"not null;comment:行索引"`
|
||||
Inputs string `json:"inputs" gorm:"type:text;comment:输入参数"`
|
||||
Status string `json:"status" gorm:"not null;default:'pending';comment:状态: pending, running, completed, failed, cancelled"`
|
||||
Result string `json:"result" gorm:"type:text;comment:处理结果"`
|
||||
Error string `json:"error" gorm:"comment:错误信息"`
|
||||
ErrorCount int `json:"error_count" gorm:"not null;default:0;comment:错误次数"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"not null;default:CURRENT_TIMESTAMP(0);comment:创建时间"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"not null;default:CURRENT_TIMESTAMP(0);comment:更新时间"`
|
||||
}
|
||||
|
||||
func (BatchWorkflow) TableName() string { return "batch_workflows_extend" }
|
||||
func (BatchWorkflowTask) TableName() string { return "batch_workflow_tasks_extend" }
|
||||
@@ -0,0 +1,44 @@
|
||||
package request
|
||||
|
||||
type WorkflowBatchProcessing struct {
|
||||
Outputs map[string]string `json:"outputs" gorm:"comment:从任务生成CSV内容"` // 从任务生成CSV内容
|
||||
}
|
||||
|
||||
// SSEEvent 表示一个SSE事件
|
||||
type SSEEvent struct {
|
||||
Event string `json:"event"`
|
||||
Data map[string]interface{} `json:"data"`
|
||||
}
|
||||
|
||||
// NodeExecution 表示节点执行信息
|
||||
type NodeExecution struct {
|
||||
ID string `json:"id"`
|
||||
NodeID string `json:"node_id"`
|
||||
NodeType string `json:"node_type"`
|
||||
Title string `json:"title"`
|
||||
Index int `json:"index"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
ElapsedTime float64 `json:"elapsed_time"`
|
||||
Inputs map[string]interface{} `json:"inputs,omitempty"`
|
||||
Outputs map[string]interface{} `json:"outputs,omitempty"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
FinishedAt int64 `json:"finished_at,omitempty"`
|
||||
}
|
||||
|
||||
// WorkflowResult 表示工作流执行结果
|
||||
type WorkflowResult struct {
|
||||
WorkflowRunID string `json:"workflow_run_id"`
|
||||
WorkflowID string `json:"workflow_id"`
|
||||
SequenceNumber int `json:"sequence_number"`
|
||||
Status string `json:"status"`
|
||||
Outputs map[string]interface{} `json:"outputs"`
|
||||
Error string `json:"error,omitempty"`
|
||||
ElapsedTime float64 `json:"elapsed_time"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
TotalSteps int `json:"total_steps"`
|
||||
ExceptionsCount int `json:"exceptions_count"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
FinishedAt int64 `json:"finished_at,omitempty"`
|
||||
Nodes []NodeExecution `json:"nodes"`
|
||||
}
|
||||
@@ -8,6 +8,7 @@ type RouterGroup struct {
|
||||
TenantsRouter
|
||||
SystemRouter
|
||||
TestRouter
|
||||
WorkflowRouter
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -18,3 +19,4 @@ var systemOAuth2Api = api.ApiGroupApp.GaiaApiGroup.SystemOAuth2Api
|
||||
var systemApi = api.ApiGroupApp.GaiaApiGroup.SystemApi
|
||||
var quotaApi = api.ApiGroupApp.GaiaApiGroup.QuotaApi
|
||||
var testApi = api.ApiGroupApp.GaiaApiGroup.TestApi
|
||||
var batchWorkflowApi = api.ApiGroupApp.GaiaApiGroup.BatchWorkflowApi
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package gaia
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type WorkflowRouter struct{}
|
||||
|
||||
// InitWorkflowRouter 初始化批量处理工作流路由
|
||||
func (w *WorkflowRouter) InitWorkflowRouter(Router *gin.RouterGroup) {
|
||||
workflowRouter := Router.Group("gaia/workflow")
|
||||
{
|
||||
// 批量处理工作流相关路由
|
||||
workflowRouter.POST("batch/processing", batchWorkflowApi.CreateBatchWorkflow) // 创建批量处理
|
||||
workflowRouter.GET("batch/list", batchWorkflowApi.GetBatchWorkflowList) // 获取最近30天的批量工作流列表
|
||||
workflowRouter.GET("batch/:id", batchWorkflowApi.GetBatchWorkflow) // 获取批量处理信息
|
||||
workflowRouter.GET("batch/:id/tasks", batchWorkflowApi.GetBatchWorkflowTasks) // 获取任务列表
|
||||
workflowRouter.GET("batch/:id/progress", batchWorkflowApi.GetBatchWorkflowProgress) // 获取进度信息
|
||||
workflowRouter.POST("batch/:id/stop", batchWorkflowApi.StopBatchWorkflow) // 停止批量处理
|
||||
workflowRouter.POST("batch/:id/retry", batchWorkflowApi.RetryBatchWorkflow) // 重试批量处理(重新开始所有任务)
|
||||
workflowRouter.POST("batch/:id/retry-failed", batchWorkflowApi.RetryFailedTasks) // 仅重试失败的任务
|
||||
workflowRouter.POST("batch/:id/resume", batchWorkflowApi.ResumeBatchWorkflow) // 恢复批量处理
|
||||
workflowRouter.GET("batch/:id/download", batchWorkflowApi.DownloadBatchWorkflowResults) // 下载结果
|
||||
|
||||
// 工作池管理相关路由
|
||||
workflowRouter.GET("worker-pool/status", batchWorkflowApi.GetWorkerPoolStatus) // 获取工作池状态
|
||||
workflowRouter.POST("worker-pool/restart", batchWorkflowApi.RestartWorkerPool) // 重启工作池
|
||||
workflowRouter.POST("worker-pool/stop", batchWorkflowApi.StopWorkerPool) // 停止工作池
|
||||
workflowRouter.POST("worker-pool/start", batchWorkflowApi.StartWorkerPool) // 启动工作池
|
||||
|
||||
// 错误计数重置相关路由
|
||||
workflowRouter.POST("batch/:id/reset-error-count", batchWorkflowApi.ResetBatchWorkflowErrorCount) // 重置批量工作流错误计数
|
||||
workflowRouter.POST("batch/reset-user-error-count", batchWorkflowApi.ResetUserErrorCount) // 重置用户所有批量工作流错误计数
|
||||
}
|
||||
}
|
||||
@@ -128,8 +128,12 @@ func RegisterUser(u system.SysUser, token string) (err error) {
|
||||
global.GVA_LOG.Debug("注册用户信息:", zap.Any("1", 1))
|
||||
var acc gaia.Account
|
||||
if err = global.GVA_DB.Where("email=?", u.Email).First(&acc).Error; err == nil {
|
||||
// 用户已存在
|
||||
global.GVA_LOG.Info(fmt.Sprintf("account %s", acc.Name))
|
||||
// 用户已存在,更新密码
|
||||
global.GVA_LOG.Info(fmt.Sprintf("account %s already exists, updating password", acc.Name))
|
||||
global.GVA_DB.Model(&acc).Updates(&map[string]interface{}{
|
||||
"password": passwordHashed,
|
||||
"password_salt": salt,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
// 默认以root执行
|
||||
@@ -178,7 +182,7 @@ func RegisterUser(u system.SysUser, token string) (err error) {
|
||||
}
|
||||
|
||||
// result
|
||||
if result, ok := bodyMap["result"]; !ok && result != "success" {
|
||||
if result, ok := bodyMap["result"]; !ok || result != "success" {
|
||||
return errors.New(fmt.Sprintf("failed to create user: %s", bodyMap["error"]))
|
||||
}
|
||||
// 修改密码
|
||||
|
||||
@@ -0,0 +1,637 @@
|
||||
package gaia
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/model/gaia/request"
|
||||
"github.com/pkg/errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/global"
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/model/gaia"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type BatchWorkflowService struct{}
|
||||
|
||||
// CreateBatchWorkflow 创建批量处理工作流
|
||||
|
||||
func (s *BatchWorkflowService) CreateBatchWorkflow(
|
||||
userId uint, installedID, fileName string, fileContent [][]string, keyNameMapping map[string]string) (
|
||||
*gaia.BatchWorkflow, error) {
|
||||
// 检查数据库连接
|
||||
if global.GVA_DB == nil {
|
||||
return nil, fmt.Errorf("数据库连接未初始化")
|
||||
}
|
||||
|
||||
// 创建批量处理记录
|
||||
keyByte, _ := json.Marshal(keyNameMapping)
|
||||
batchWorkflow := &gaia.BatchWorkflow{
|
||||
ProcessedRows: 0,
|
||||
UserID: userId,
|
||||
FileName: fileName,
|
||||
Status: "pending",
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
InstalledID: installedID,
|
||||
KeyName: string(keyByte),
|
||||
ID: uuid.New().String(),
|
||||
TotalRows: 0, // 先设为0,后面会更新为实际有效行数
|
||||
}
|
||||
|
||||
// 保存到数据库
|
||||
if err := global.GVA_DB.Create(batchWorkflow).Error; err != nil {
|
||||
return nil, fmt.Errorf("保存批量处理记录失败: %v", err)
|
||||
}
|
||||
|
||||
// 创建任务记录
|
||||
headers := fileContent[0]
|
||||
if len(headers) > 0 {
|
||||
// 去除UTF-8 BOM
|
||||
headers[0] = strings.TrimPrefix(headers[0], "\uFEFF")
|
||||
}
|
||||
dataRows := fileContent[1:]
|
||||
|
||||
validRowCount := 0 // 记录有效行数
|
||||
for i, row := range dataRows {
|
||||
// 构建输入参数
|
||||
inputs := make(map[string]string)
|
||||
hasNonEmptyValue := false // 检查是否有非空值
|
||||
|
||||
for j, value := range row {
|
||||
if j < len(headers) {
|
||||
headerName := headers[j]
|
||||
// 去除首尾空格
|
||||
value = strings.TrimSpace(value)
|
||||
|
||||
// 如果有key-name映射,使用映射后的key,否则使用原始header
|
||||
if keyNameMapping != nil {
|
||||
if key, exists := keyNameMapping[headerName]; exists {
|
||||
inputs[key] = value
|
||||
} else {
|
||||
inputs[headerName] = value
|
||||
}
|
||||
} else {
|
||||
inputs[headerName] = value
|
||||
}
|
||||
|
||||
// 检查是否有非空值
|
||||
if value != "" {
|
||||
hasNonEmptyValue = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 如果所有字段都为空,跳过这一行
|
||||
if !hasNonEmptyValue {
|
||||
global.GVA_LOG.Info(fmt.Sprintf("跳过空值行,行索引: %d", i+1))
|
||||
continue
|
||||
}
|
||||
|
||||
validRowCount++
|
||||
inputsJSON, _ := json.Marshal(inputs)
|
||||
|
||||
task := &gaia.BatchWorkflowTask{
|
||||
ID: uuid.New().String(),
|
||||
BatchWorkflowID: batchWorkflow.ID,
|
||||
RowIndex: i + 1,
|
||||
Inputs: string(inputsJSON),
|
||||
Status: "pending",
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
|
||||
if err := global.GVA_DB.Create(task).Error; err != nil {
|
||||
return nil, fmt.Errorf("创建任务记录失败: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 更新批量处理记录的总行数为实际有效行数
|
||||
if err := global.GVA_DB.Model(batchWorkflow).Update("total_rows", validRowCount).Error; err != nil {
|
||||
global.GVA_LOG.Error(fmt.Sprintf("更新总行数失败: %v", err))
|
||||
}
|
||||
|
||||
global.GVA_LOG.Info(fmt.Sprintf("批量工作流 %s 创建完成,原始行数: %d,有效行数: %d",
|
||||
batchWorkflow.ID, len(fileContent)-1, validRowCount))
|
||||
|
||||
// 任务已创建,工作池会自动处理
|
||||
// 确保工作池在运行
|
||||
if pool := GetWorkerPool(); pool == nil || !pool.IsRunning() {
|
||||
global.GVA_LOG.Warn("工作池未运行,尝试重新启动")
|
||||
InitWorkerPool(global.GVA_CONFIG.System.WorkFlowNumber) // 默认5个worker
|
||||
}
|
||||
|
||||
// 更新批处理工作流状态为处理中
|
||||
if err := global.GVA_DB.Model(batchWorkflow).Update("status", "processing").Error; err != nil {
|
||||
global.GVA_LOG.Error(fmt.Sprintf("更新批处理工作流状态失败: %v", err))
|
||||
}
|
||||
|
||||
return batchWorkflow, nil
|
||||
}
|
||||
|
||||
// parseSSEStream 解析SSE流并返回最终结果
|
||||
func (s *BatchWorkflowService) parseSSEStream(body []byte) (*request.WorkflowResult, error) {
|
||||
lines := strings.Split(string(body), "\n")
|
||||
result := &request.WorkflowResult{
|
||||
Nodes: make([]request.NodeExecution, 0),
|
||||
}
|
||||
nodeMap := make(map[string]*request.NodeExecution) // 用于跟踪节点状态
|
||||
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || !strings.HasPrefix(line, "data: ") {
|
||||
continue
|
||||
}
|
||||
|
||||
// 移除 "data: " 前缀
|
||||
jsonStr := strings.TrimPrefix(line, "data: ")
|
||||
|
||||
// 解析JSON
|
||||
var event map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(jsonStr), &event); err != nil {
|
||||
continue // 跳过无法解析的行
|
||||
}
|
||||
|
||||
eventType, ok := event["event"].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// 兼容新旧格式:如果有data字段则使用data,否则使用顶层数据
|
||||
var data map[string]interface{}
|
||||
if dataField, hasData := event["data"].(map[string]interface{}); hasData {
|
||||
// 旧格式:事件数据在data字段中
|
||||
data = dataField
|
||||
} else {
|
||||
// 新格式:事件数据在顶层
|
||||
data = event
|
||||
}
|
||||
|
||||
switch eventType {
|
||||
case "workflow_started":
|
||||
if workflowRunID, ok := data["id"].(string); ok {
|
||||
result.WorkflowRunID = workflowRunID
|
||||
}
|
||||
if workflowID, ok := data["workflow_id"].(string); ok {
|
||||
result.WorkflowID = workflowID
|
||||
}
|
||||
if sequenceNumber, ok := data["sequence_number"].(float64); ok {
|
||||
result.SequenceNumber = int(sequenceNumber)
|
||||
}
|
||||
if createdAt, ok := data["created_at"].(float64); ok {
|
||||
result.CreatedAt = int64(createdAt)
|
||||
}
|
||||
|
||||
case "node_started":
|
||||
nodeExecution := &request.NodeExecution{}
|
||||
if id, ok := data["id"].(string); ok {
|
||||
nodeExecution.ID = id
|
||||
}
|
||||
if nodeID, ok := data["node_id"].(string); ok {
|
||||
nodeExecution.NodeID = nodeID
|
||||
}
|
||||
if nodeType, ok := data["node_type"].(string); ok {
|
||||
nodeExecution.NodeType = nodeType
|
||||
}
|
||||
if title, ok := data["title"].(string); ok {
|
||||
nodeExecution.Title = title
|
||||
}
|
||||
if index, ok := data["index"].(float64); ok {
|
||||
nodeExecution.Index = int(index)
|
||||
}
|
||||
if inputs, ok := data["inputs"].(map[string]interface{}); ok {
|
||||
nodeExecution.Inputs = inputs
|
||||
}
|
||||
if createdAt, ok := data["created_at"].(float64); ok {
|
||||
nodeExecution.CreatedAt = int64(createdAt)
|
||||
}
|
||||
|
||||
nodeMap[nodeExecution.ID] = nodeExecution
|
||||
|
||||
case "node_finished":
|
||||
nodeID, ok := data["id"].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
node, exists := nodeMap[nodeID]
|
||||
if !exists {
|
||||
// 如果没有找到对应的开始节点,创建一个新的
|
||||
node = &request.NodeExecution{}
|
||||
if id, ok := data["id"].(string); ok {
|
||||
node.ID = id
|
||||
}
|
||||
if nodeIDStr, ok := data["node_id"].(string); ok {
|
||||
node.NodeID = nodeIDStr
|
||||
}
|
||||
if nodeType, ok := data["node_type"].(string); ok {
|
||||
node.NodeType = nodeType
|
||||
}
|
||||
if title, ok := data["title"].(string); ok {
|
||||
node.Title = title
|
||||
}
|
||||
if index, ok := data["index"].(float64); ok {
|
||||
node.Index = int(index)
|
||||
}
|
||||
nodeMap[nodeID] = node
|
||||
}
|
||||
|
||||
// 更新节点完成信息
|
||||
if status, ok := data["status"].(string); ok {
|
||||
node.Status = status
|
||||
}
|
||||
if errorMsg, ok := data["error"].(string); ok && errorMsg != "" {
|
||||
node.Error = errorMsg
|
||||
}
|
||||
if elapsedTime, ok := data["elapsed_time"].(float64); ok {
|
||||
node.ElapsedTime = elapsedTime
|
||||
}
|
||||
if outputs, ok := data["outputs"].(map[string]interface{}); ok {
|
||||
node.Outputs = outputs
|
||||
}
|
||||
if finishedAt, ok := data["finished_at"].(float64); ok {
|
||||
node.FinishedAt = int64(finishedAt)
|
||||
}
|
||||
|
||||
case "workflow_finished":
|
||||
if status, ok := data["status"].(string); ok {
|
||||
result.Status = status
|
||||
}
|
||||
if outputs, ok := data["outputs"].(map[string]interface{}); ok {
|
||||
result.Outputs = outputs
|
||||
}
|
||||
if errorMsg, ok := data["error"].(string); ok {
|
||||
result.Error = errorMsg
|
||||
}
|
||||
if elapsedTime, ok := data["elapsed_time"].(float64); ok {
|
||||
result.ElapsedTime = elapsedTime
|
||||
}
|
||||
if totalTokens, ok := data["total_tokens"].(float64); ok {
|
||||
result.TotalTokens = int(totalTokens)
|
||||
}
|
||||
if totalSteps, ok := data["total_steps"].(float64); ok {
|
||||
result.TotalSteps = int(totalSteps)
|
||||
}
|
||||
if exceptionsCount, ok := data["exceptions_count"].(float64); ok {
|
||||
result.ExceptionsCount = int(exceptionsCount)
|
||||
}
|
||||
if finishedAt, ok := data["finished_at"].(float64); ok {
|
||||
result.FinishedAt = int64(finishedAt)
|
||||
}
|
||||
|
||||
case "message":
|
||||
// 处理新的message事件格式,将answer字段填充到outputs.text中
|
||||
if answer, ok := data["answer"].(string); ok && answer != "" {
|
||||
// 如果result.Outputs为空,初始化它
|
||||
if result.Outputs == nil {
|
||||
result.Outputs = make(map[string]interface{})
|
||||
}
|
||||
if value, okText := result.Outputs["text"]; okText {
|
||||
result.Outputs["text"] = value.(string) + answer
|
||||
} else {
|
||||
result.Outputs["text"] = answer
|
||||
}
|
||||
}
|
||||
// 同时设置其他相关字段
|
||||
if messageID, ok := data["message_id"].(string); ok {
|
||||
result.WorkflowRunID = messageID
|
||||
}
|
||||
if createdAt, ok := data["created_at"].(float64); ok {
|
||||
result.CreatedAt = int64(createdAt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 将节点按照index排序并添加到结果中
|
||||
for _, node := range nodeMap {
|
||||
result.Nodes = append(result.Nodes, *node)
|
||||
}
|
||||
|
||||
// 按index排序
|
||||
for i := 0; i < len(result.Nodes)-1; i++ {
|
||||
for j := i + 1; j < len(result.Nodes); j++ {
|
||||
if result.Nodes[i].Index > result.Nodes[j].Index {
|
||||
result.Nodes[i], result.Nodes[j] = result.Nodes[j], result.Nodes[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// callDifyAPI 调用Dify API
|
||||
func (s *BatchWorkflowService) callDifyAPI(
|
||||
installedID, userToken string, inputs map[string]string) (string, error) {
|
||||
|
||||
var err error
|
||||
var requestBodyJSON []byte
|
||||
if requestBodyJSON, err = json.Marshal(&map[string]interface{}{
|
||||
"inputs": inputs,
|
||||
"response_mode": "streaming",
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var url string
|
||||
var mode sql.NullString
|
||||
if err = global.GVA_DB.Raw("SELECT b.mode FROM installed_apps as a, apps as b WHERE a.app_id=b.id AND a.id = ?", installedID).Scan(&mode).Error; err != nil {
|
||||
return "", err
|
||||
}
|
||||
// 区分model
|
||||
if mode.String == "workflow" {
|
||||
url = "%s/console/api/installed-apps/%s/workflows/run"
|
||||
} else if mode.String == "completion" {
|
||||
url = "%s/console/api/installed-apps/%s/completion-messages"
|
||||
} else {
|
||||
return "", errors.New(fmt.Sprintf("Unsupported dify API call: %s", mode.String))
|
||||
}
|
||||
// 创建HTTP请求
|
||||
req, err := http.NewRequest("POST", fmt.Sprintf(
|
||||
url, global.GVA_CONFIG.Gaia.Url, installedID), strings.NewReader(string(requestBodyJSON)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// 设置请求头
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+userToken)
|
||||
req.Header.Set("Accept", "text/event-stream") // 接受SSE流
|
||||
|
||||
// 发送请求
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// 读取响应
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("API请求失败,状态码: %d, 响应: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// 解析SSE流
|
||||
result, err := s.parseSSEStream(body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("解析SSE流失败: %v", err)
|
||||
}
|
||||
|
||||
// 将结果转换为JSON字符串返回
|
||||
resultJSON, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("序列化结果失败: %v", err)
|
||||
}
|
||||
|
||||
return string(resultJSON), nil
|
||||
}
|
||||
|
||||
// GetBatchWorkflow 获取批量处理信息
|
||||
func (s *BatchWorkflowService) GetBatchWorkflow(id string) (*gaia.BatchWorkflow, error) {
|
||||
if global.GVA_DB == nil {
|
||||
return nil, fmt.Errorf("数据库连接未初始化")
|
||||
}
|
||||
|
||||
var batchWorkflow gaia.BatchWorkflow
|
||||
if err := global.GVA_DB.Where("id = ?", id).First(&batchWorkflow).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &batchWorkflow, nil
|
||||
}
|
||||
|
||||
// GetBatchWorkflowTasks 获取批量处理的任务列表
|
||||
func (s *BatchWorkflowService) GetBatchWorkflowTasks(batchWorkflowID string) ([]gaia.BatchWorkflowTask, error) {
|
||||
if global.GVA_DB == nil {
|
||||
return nil, fmt.Errorf("数据库连接未初始化")
|
||||
}
|
||||
|
||||
var tasks []gaia.BatchWorkflowTask
|
||||
if err := global.GVA_DB.Where("batch_workflow_id = ?", batchWorkflowID).Order("row_index").Find(&tasks).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
// StopBatchWorkflow 停止批量处理
|
||||
func (s *BatchWorkflowService) StopBatchWorkflow(id string) error {
|
||||
if global.GVA_DB == nil {
|
||||
return fmt.Errorf("数据库连接未初始化")
|
||||
}
|
||||
|
||||
return global.GVA_DB.Model(&gaia.BatchWorkflow{}).Where("id = ?", id).Update("status", "stopped").Error
|
||||
}
|
||||
|
||||
// RetryFailedTasks 仅重试失败的任务
|
||||
func (s *BatchWorkflowService) RetryFailedTasks(id string) error {
|
||||
if global.GVA_DB == nil {
|
||||
return fmt.Errorf("数据库连接未初始化")
|
||||
}
|
||||
|
||||
// 只重置失败的任务为待处理状态,保留已完成的任务
|
||||
errorCount := 0
|
||||
if err := global.GVA_DB.Model(&gaia.BatchWorkflowTask{}).Where(
|
||||
"batch_workflow_id = ? AND status IN ?", id, []string{"failed", "queued", "running"}).Updates(
|
||||
map[string]interface{}{
|
||||
"status": "pending",
|
||||
"error": "",
|
||||
"error_count": &errorCount,
|
||||
"updated_at": time.Now(),
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 重新计算已处理行数
|
||||
var completedCount int64
|
||||
global.GVA_DB.Model(&gaia.BatchWorkflowTask{}).Where(
|
||||
"batch_workflow_id = ? AND status = ?", id, "completed").Count(&completedCount)
|
||||
|
||||
// 重置批量处理状态
|
||||
if err := global.GVA_DB.Model(&gaia.BatchWorkflow{}).Where("id = ?", id).Updates(map[string]interface{}{
|
||||
"status": "pending",
|
||||
"processed_rows": completedCount,
|
||||
"error": "",
|
||||
"updated_at": time.Now(),
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
global.GVA_LOG.Info(fmt.Sprintf("批量工作流 %s 失败任务重试已启动,工作池将自动处理待处理任务", id))
|
||||
return nil
|
||||
}
|
||||
|
||||
// RetryBatchWorkflow 重试批量处理
|
||||
func (s *BatchWorkflowService) RetryBatchWorkflow(id string) error {
|
||||
if global.GVA_DB == nil {
|
||||
return fmt.Errorf("数据库连接未初始化")
|
||||
}
|
||||
|
||||
// 重置所有失败的任务为待处理状态
|
||||
if err := global.GVA_DB.Model(&gaia.BatchWorkflowTask{}).Where("batch_workflow_id = ? AND status IN ?", id, []string{"failed", "queued", "running"}).Updates(map[string]interface{}{
|
||||
"status": "pending",
|
||||
"error": "",
|
||||
"updated_at": time.Now(),
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 重新计算已处理行数
|
||||
var completedCount int64
|
||||
global.GVA_DB.Model(&gaia.BatchWorkflowTask{}).Where("batch_workflow_id = ? AND status = ?", id, "completed").Count(&completedCount)
|
||||
|
||||
// 重置批量处理状态
|
||||
if err := global.GVA_DB.Model(&gaia.BatchWorkflow{}).Where("id = ?", id).Updates(map[string]interface{}{
|
||||
"status": "processing",
|
||||
"processed_rows": completedCount,
|
||||
"error": "",
|
||||
"updated_at": time.Now(),
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 确保工作池在运行
|
||||
if pool := GetWorkerPool(); pool == nil || !pool.IsRunning() {
|
||||
global.GVA_LOG.Warn("工作池未运行,尝试重新启动")
|
||||
InitWorkerPool(global.GVA_CONFIG.System.WorkFlowNumber)
|
||||
}
|
||||
|
||||
global.GVA_LOG.Info(fmt.Sprintf("批量工作流 %s 重试已启动,工作池将自动处理待处理任务", id))
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResumeBatchWorkflow 恢复批量处理
|
||||
func (s *BatchWorkflowService) ResumeBatchWorkflow(id string) error {
|
||||
if global.GVA_DB == nil {
|
||||
return fmt.Errorf("数据库连接未初始化")
|
||||
}
|
||||
|
||||
// 检查批量工作流是否存在
|
||||
var batchWorkflow gaia.BatchWorkflow
|
||||
if err := global.GVA_DB.Where("id = ?", id).First(&batchWorkflow).Error; err != nil {
|
||||
return fmt.Errorf("批量工作流不存在: %v", err)
|
||||
}
|
||||
|
||||
// 检查批量工作流状态是否为stopped
|
||||
if batchWorkflow.Status != "stopped" {
|
||||
return fmt.Errorf("只能恢复已停止的批量处理")
|
||||
}
|
||||
|
||||
// 检查是否有可恢复的任务(pending 或 cancelled 状态)
|
||||
var resumableTasks int64
|
||||
global.GVA_DB.Model(&gaia.BatchWorkflowTask{}).Where("batch_workflow_id = ? AND status IN (?)", id, []string{"pending", "cancelled"}).Count(&resumableTasks)
|
||||
|
||||
if resumableTasks == 0 {
|
||||
return fmt.Errorf("没有可恢复的任务")
|
||||
}
|
||||
|
||||
// 将cancelled状态的任务恢复为pending状态
|
||||
if err := global.GVA_DB.Model(&gaia.BatchWorkflowTask{}).
|
||||
Where("batch_workflow_id = ? AND status = ?", id, "cancelled").
|
||||
Updates(map[string]interface{}{
|
||||
"status": "pending",
|
||||
"updated_at": time.Now(),
|
||||
}).Error; err != nil {
|
||||
return fmt.Errorf("恢复取消的任务失败: %v", err)
|
||||
}
|
||||
|
||||
// 更新批量工作流状态为处理中
|
||||
if err := global.GVA_DB.Model(&gaia.BatchWorkflow{}).Where("id = ?", id).Updates(map[string]interface{}{
|
||||
"status": "processing",
|
||||
"updated_at": time.Now(),
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 确保工作池在运行
|
||||
if pool := GetWorkerPool(); pool == nil || !pool.IsRunning() {
|
||||
global.GVA_LOG.Warn("工作池未运行,尝试重新启动")
|
||||
InitWorkerPool(global.GVA_CONFIG.System.WorkFlowNumber)
|
||||
}
|
||||
|
||||
global.GVA_LOG.Info(fmt.Sprintf("批量工作流 %s 恢复已启动,工作池将自动处理待处理任务", id))
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBatchWorkflowProgress 获取批量处理进度
|
||||
func (s *BatchWorkflowService) GetBatchWorkflowProgress(id string) (map[string]interface{}, error) {
|
||||
if global.GVA_DB == nil {
|
||||
return nil, fmt.Errorf("数据库连接未初始化")
|
||||
}
|
||||
|
||||
var batchWorkflow gaia.BatchWorkflow
|
||||
if err := global.GVA_DB.Where("id = ?", id).First(&batchWorkflow).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 统计各种状态的任务数量
|
||||
var pendingCount, queuedCount, runningCount, completedCount, failedCount int64
|
||||
global.GVA_DB.Model(&gaia.BatchWorkflowTask{}).Where("batch_workflow_id = ? AND status = ?", id, "pending").Count(&pendingCount)
|
||||
global.GVA_DB.Model(&gaia.BatchWorkflowTask{}).Where("batch_workflow_id = ? AND status = ?", id, "queued").Count(&queuedCount)
|
||||
global.GVA_DB.Model(&gaia.BatchWorkflowTask{}).Where("batch_workflow_id = ? AND status = ?", id, "running").Count(&runningCount)
|
||||
global.GVA_DB.Model(&gaia.BatchWorkflowTask{}).Where("batch_workflow_id = ? AND status = ?", id, "completed").Count(&completedCount)
|
||||
global.GVA_DB.Model(&gaia.BatchWorkflowTask{}).Where("batch_workflow_id = ? AND status = ?", id, "failed").Count(&failedCount)
|
||||
|
||||
progress := float64(completedCount) / float64(batchWorkflow.TotalRows) * 100
|
||||
|
||||
// 获取工作池状态
|
||||
var workerPoolStatus map[string]interface{}
|
||||
if pool := GetWorkerPool(); pool != nil {
|
||||
workerPoolStatus = pool.GetStatus()
|
||||
} else {
|
||||
workerPoolStatus = map[string]interface{}{
|
||||
"running": false,
|
||||
"workers": 0,
|
||||
"queue_length": 0,
|
||||
}
|
||||
}
|
||||
|
||||
// 获取错误信息 - 从批量工作流本身和失败的任务中获取
|
||||
var errorInfo string
|
||||
if batchWorkflow.Error != "" {
|
||||
errorInfo = batchWorkflow.Error
|
||||
} else if failedCount > 0 {
|
||||
// 如果有失败的任务,获取第一个失败任务的错误信息作为代表
|
||||
var failedTask gaia.BatchWorkflowTask
|
||||
if err := global.GVA_DB.Where("batch_workflow_id = ? AND status = ?", id, "failed").First(&failedTask).Error; err == nil && failedTask.Error != "" {
|
||||
errorInfo = failedTask.Error
|
||||
}
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"id": batchWorkflow.ID,
|
||||
"status": batchWorkflow.Status,
|
||||
"total_rows": batchWorkflow.TotalRows,
|
||||
"processed_rows": completedCount, // 使用实时统计值确保与progress一致
|
||||
"progress": progress,
|
||||
"pending_count": pendingCount,
|
||||
"queued_count": queuedCount,
|
||||
"running_count": runningCount,
|
||||
"completed_count": completedCount,
|
||||
"failed_count": failedCount,
|
||||
"error": errorInfo, // 添加错误信息
|
||||
"worker_pool_status": workerPoolStatus,
|
||||
"created_at": batchWorkflow.CreatedAt,
|
||||
"updated_at": batchWorkflow.UpdatedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetWorkerPool 获取全局工作池
|
||||
func (s *BatchWorkflowService) GetWorkerPool() *WorkerPool {
|
||||
return GetWorkerPool()
|
||||
}
|
||||
|
||||
// InitWorkerPool 初始化工作池
|
||||
func (s *BatchWorkflowService) InitWorkerPool(workers int) {
|
||||
InitWorkerPool(workers)
|
||||
}
|
||||
|
||||
// StopWorkerPool 停止工作池
|
||||
func (s *BatchWorkflowService) StopWorkerPool() {
|
||||
StopWorkerPool()
|
||||
}
|
||||
@@ -6,4 +6,5 @@ type ServiceGroup struct {
|
||||
QuotaService
|
||||
TenantsService
|
||||
TestService
|
||||
BatchWorkflowService
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -39,9 +39,22 @@ var UserServiceApp = new(UserService)
|
||||
// @return: err error, userInter *model.SysUser
|
||||
func (userService *UserService) Register(u system.SysUser, token string) (userInter system.SysUser, err error) {
|
||||
var user system.SysUser
|
||||
// 首先检查email是否已注册
|
||||
if !errors.Is(global.GVA_DB.Where("email = ?", u.Email).First(&user).Error, gorm.ErrRecordNotFound) {
|
||||
global.GVA_LOG.Info(fmt.Sprintf("用户email已存在: %s", u.Email))
|
||||
return userInter, errors.New("用户名已注册")
|
||||
}
|
||||
|
||||
// 如果传入了UUID,检查UUID是否已存在
|
||||
if u.UUID != uuid.Nil {
|
||||
var existingUser system.SysUser
|
||||
if !errors.Is(global.GVA_DB.Where("uuid = ?", u.UUID).First(&existingUser).Error, gorm.ErrRecordNotFound) {
|
||||
global.GVA_LOG.Info(fmt.Sprintf("用户UUID已存在: %s, email: %s", u.UUID, u.Email))
|
||||
// UUID已存在,返回已存在的用户而不是报错(用于SyncUser场景)
|
||||
return existingUser, nil
|
||||
}
|
||||
}
|
||||
|
||||
global.GVA_LOG.Debug("注册用户信息:", zap.Any("1", 1))
|
||||
|
||||
// Extend Start: Gaia Register User
|
||||
@@ -50,9 +63,18 @@ func (userService *UserService) Register(u system.SysUser, token string) (userIn
|
||||
}
|
||||
// Extend Stop: Gaia Register User
|
||||
|
||||
// 再次检查email是否已注册(防止并发创建)
|
||||
if !errors.Is(global.GVA_DB.Where("email = ?", u.Email).First(&user).Error, gorm.ErrRecordNotFound) {
|
||||
global.GVA_LOG.Info(fmt.Sprintf("并发检测:用户email已被创建: %s", u.Email))
|
||||
return user, nil
|
||||
}
|
||||
|
||||
// 否则 附加uuid 密码hash加密 注册
|
||||
u.Password = utils.BcryptHash(u.Password)
|
||||
u.UUID = uuid.Must(uuid.NewV4())
|
||||
// 如果没有设置UUID,才生成新的UUID
|
||||
if u.UUID == uuid.Nil {
|
||||
u.UUID = uuid.Must(uuid.NewV4())
|
||||
}
|
||||
err = global.GVA_DB.Create(&u).Error
|
||||
return u, err
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package system
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/global"
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/model/gaia"
|
||||
@@ -11,6 +12,9 @@ import (
|
||||
"github.com/gofrs/uuid/v5"
|
||||
)
|
||||
|
||||
// 全局互斥锁,防止SyncUser并发执行
|
||||
var syncUserMutex sync.Mutex
|
||||
|
||||
//@author: [piexlmax](https://github.com/piexlmax)
|
||||
//@function: Register
|
||||
//@description: 用户注册
|
||||
@@ -47,6 +51,10 @@ func (userService *UserExtendService) OaLogin(u *system.SysUser) (userInter *sys
|
||||
// @param: u *model.SysUser
|
||||
// @return: err error, userInter *model.SysUser
|
||||
func (userService *UserExtendService) SyncUser() {
|
||||
// 使用互斥锁防止并发执行
|
||||
syncUserMutex.Lock()
|
||||
defer syncUserMutex.Unlock()
|
||||
|
||||
// init
|
||||
var err error
|
||||
var isInit = true
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package gaia
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/model/gaia"
|
||||
"github.com/flipped-aurora/gin-vue-admin/server/service/system"
|
||||
"github.com/gofrs/uuid/v5"
|
||||
"github.com/pkg/errors"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const initOrderForwardingExtend = system.InitOrderInternal + 1
|
||||
|
||||
type initForwardingExtend struct{}
|
||||
|
||||
// auto run
|
||||
func init() {
|
||||
system.RegisterInit(initOrderForwardingExtend, &initForwardingExtend{})
|
||||
}
|
||||
|
||||
func (i *initForwardingExtend) MigrateTable(ctx context.Context) (context.Context, error) {
|
||||
db, ok := ctx.Value("db").(*gorm.DB)
|
||||
if !ok {
|
||||
return ctx, system.ErrMissingDBContext
|
||||
}
|
||||
return ctx, db.AutoMigrate(&gaia.ForwardingExtend{})
|
||||
}
|
||||
|
||||
func (i *initForwardingExtend) TableCreated(ctx context.Context) bool {
|
||||
db, ok := ctx.Value("db").(*gorm.DB)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return db.Migrator().HasTable(&gaia.ForwardingExtend{})
|
||||
}
|
||||
|
||||
func (i initForwardingExtend) InitializerName() string {
|
||||
return gaia.ForwardingExtend{}.TableName()
|
||||
}
|
||||
|
||||
func (i *initForwardingExtend) InitializeData(ctx context.Context) (context.Context, error) {
|
||||
db, ok := ctx.Value("db").(*gorm.DB)
|
||||
if !ok {
|
||||
return ctx, system.ErrMissingDBContext
|
||||
}
|
||||
|
||||
// 使用指定的 UUID
|
||||
id, err := uuid.FromString("dbb08cae-2118-469c-a991-0c8f3f2515da")
|
||||
if err != nil {
|
||||
return ctx, errors.Wrap(err, "解析 UUID 失败")
|
||||
}
|
||||
|
||||
entities := []gaia.ForwardingExtend{
|
||||
{
|
||||
ID: id,
|
||||
Path: "workflow",
|
||||
Address: "http://admin-server:8888/gaia/workflow/",
|
||||
Header: "[]",
|
||||
Description: "",
|
||||
},
|
||||
}
|
||||
|
||||
if err := db.Create(&entities).Error; err != nil {
|
||||
return ctx, errors.Wrap(err, gaia.ForwardingExtend{}.TableName()+"表数据初始化失败!")
|
||||
}
|
||||
|
||||
next := context.WithValue(ctx, i.InitializerName(), entities)
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func (i *initForwardingExtend) DataInserted(ctx context.Context) bool {
|
||||
db, ok := ctx.Value("db").(*gorm.DB)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
// 检查是否存在指定的记录
|
||||
if errors.Is(db.Where("id = ?", "dbb08cae-2118-469c-a991-0c8f3f2515da").
|
||||
First(&gaia.ForwardingExtend{}).Error, gorm.ErrRecordNotFound) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -207,6 +207,19 @@ func (i *initApi) InitializeData(ctx context.Context) (context.Context, error) {
|
||||
{ApiGroup: "应用集成配置", Method: "GET", Path: "/gaia/system/oauth2", Description: "设置OAuth2配置"},
|
||||
{ApiGroup: "应用集成配置", Method: "POST", Path: "/gaia/system/oauth2", Description: "获取OAuth2集成配置"},
|
||||
// Extend Stop: oauth2
|
||||
|
||||
// Extend Start: batch workflow
|
||||
{ApiGroup: "批量处理工作流", Method: "POST", Path: "/gaia/workflow/batch/processing", Description: "创建批量处理"},
|
||||
{ApiGroup: "批量处理工作流", Method: "GET", Path: "/gaia/workflow/batch/list", Description: "获取最近30天的批量工作流列表"},
|
||||
{ApiGroup: "批量处理工作流", Method: "GET", Path: "/gaia/workflow/batch/:id", Description: "获取批量处理信息"},
|
||||
{ApiGroup: "批量处理工作流", Method: "GET", Path: "/gaia/workflow/batch/:id/tasks", Description: "获取任务列表"},
|
||||
{ApiGroup: "批量处理工作流", Method: "GET", Path: "/gaia/workflow/batch/:id/progress", Description: "获取进度信息"},
|
||||
{ApiGroup: "批量处理工作流", Method: "POST", Path: "/gaia/workflow/batch/:id/stop", Description: "停止批量处理"},
|
||||
{ApiGroup: "批量处理工作流", Method: "POST", Path: "/gaia/workflow/batch/:id/retry", Description: "重试批量处理(重新开始所有任务)"},
|
||||
{ApiGroup: "批量处理工作流", Method: "POST", Path: "/gaia/workflow/batch/:id/retry-failed", Description: "仅重试失败的任务"},
|
||||
{ApiGroup: "批量处理工作流", Method: "POST", Path: "/gaia/workflow/batch/:id/resume", Description: "恢复批量处理"},
|
||||
{ApiGroup: "批量处理工作流", Method: "GET", Path: "/gaia/workflow/batch/:id/download", Description: "下载结果"},
|
||||
// Extend Stop: batch workflow
|
||||
}
|
||||
if err := db.Create(&entities).Error; err != nil {
|
||||
return ctx, errors.Wrap(err, sysModel.SysApi{}.TableName()+"表数据初始化失败!")
|
||||
|
||||
@@ -293,6 +293,48 @@ func (i *initCasbin) InitializeData(ctx context.Context) (context.Context, error
|
||||
{Ptype: "p", V0: "888", V1: "/gaia/system/oauth2", V2: "GET"},
|
||||
{Ptype: "p", V0: "888", V1: "/gaia/system/oauth2", V2: "POST"},
|
||||
// Extend Stop: oauth2
|
||||
|
||||
// Extend Start: batch workflow
|
||||
{Ptype: "p", V0: "888", V1: "/gaia/workflow/batch/processing", V2: "POST"},
|
||||
{Ptype: "p", V0: "888", V1: "/gaia/workflow/batch/:id", V2: "GET"},
|
||||
{Ptype: "p", V0: "888", V1: "/gaia/workflow/batch/:id/tasks", V2: "GET"},
|
||||
{Ptype: "p", V0: "888", V1: "/gaia/workflow/batch/:id/progress", V2: "GET"},
|
||||
{Ptype: "p", V0: "888", V1: "/gaia/workflow/batch/:id/stop", V2: "POST"},
|
||||
{Ptype: "p", V0: "888", V1: "/gaia/workflow/batch/:id/retry", V2: "POST"},
|
||||
{Ptype: "p", V0: "888", V1: "/gaia/workflow/batch/:id/retry-failed", V2: "POST"},
|
||||
{Ptype: "p", V0: "888", V1: "/gaia/workflow/batch/:id/resume", V2: "POST"},
|
||||
{Ptype: "p", V0: "888", V1: "/gaia/workflow/batch/:id/download", V2: "GET"},
|
||||
|
||||
{Ptype: "p", V0: "8881", V1: "/gaia/workflow/batch/processing", V2: "POST"},
|
||||
{Ptype: "p", V0: "8881", V1: "/gaia/workflow/batch/:id", V2: "GET"},
|
||||
{Ptype: "p", V0: "8881", V1: "/gaia/workflow/batch/:id/tasks", V2: "GET"},
|
||||
{Ptype: "p", V0: "8881", V1: "/gaia/workflow/batch/:id/progress", V2: "GET"},
|
||||
{Ptype: "p", V0: "8881", V1: "/gaia/workflow/batch/:id/stop", V2: "POST"},
|
||||
{Ptype: "p", V0: "8881", V1: "/gaia/workflow/batch/:id/retry", V2: "POST"},
|
||||
{Ptype: "p", V0: "8881", V1: "/gaia/workflow/batch/:id/retry-failed", V2: "POST"},
|
||||
{Ptype: "p", V0: "8881", V1: "/gaia/workflow/batch/:id/resume", V2: "POST"},
|
||||
{Ptype: "p", V0: "8881", V1: "/gaia/workflow/batch/:id/download", V2: "GET"},
|
||||
|
||||
{Ptype: "p", V0: "9528", V1: "/gaia/workflow/batch/processing", V2: "POST"},
|
||||
{Ptype: "p", V0: "9528", V1: "/gaia/workflow/batch/:id", V2: "GET"},
|
||||
{Ptype: "p", V0: "9528", V1: "/gaia/workflow/batch/:id/tasks", V2: "GET"},
|
||||
{Ptype: "p", V0: "9528", V1: "/gaia/workflow/batch/:id/progress", V2: "GET"},
|
||||
{Ptype: "p", V0: "9528", V1: "/gaia/workflow/batch/:id/stop", V2: "POST"},
|
||||
{Ptype: "p", V0: "9528", V1: "/gaia/workflow/batch/:id/retry", V2: "POST"},
|
||||
{Ptype: "p", V0: "9528", V1: "/gaia/workflow/batch/:id/retry-failed", V2: "POST"},
|
||||
{Ptype: "p", V0: "9528", V1: "/gaia/workflow/batch/:id/resume", V2: "POST"},
|
||||
{Ptype: "p", V0: "9528", V1: "/gaia/workflow/batch/:id/download", V2: "GET"},
|
||||
|
||||
{Ptype: "p", V0: "1", V1: "/gaia/workflow/batch/processing", V2: "POST"},
|
||||
{Ptype: "p", V0: "1", V1: "/gaia/workflow/batch/:id", V2: "GET"},
|
||||
{Ptype: "p", V0: "1", V1: "/gaia/workflow/batch/:id/tasks", V2: "GET"},
|
||||
{Ptype: "p", V0: "1", V1: "/gaia/workflow/batch/:id/progress", V2: "GET"},
|
||||
{Ptype: "p", V0: "1", V1: "/gaia/workflow/batch/:id/stop", V2: "POST"},
|
||||
{Ptype: "p", V0: "1", V1: "/gaia/workflow/batch/:id/retry", V2: "POST"},
|
||||
{Ptype: "p", V0: "1", V1: "/gaia/workflow/batch/:id/retry-failed", V2: "POST"},
|
||||
{Ptype: "p", V0: "1", V1: "/gaia/workflow/batch/:id/resume", V2: "POST"},
|
||||
{Ptype: "p", V0: "1", V1: "/gaia/workflow/batch/:id/download", V2: "GET"},
|
||||
// Extend Stop: batch workflow
|
||||
}
|
||||
if err := db.Create(&entities).Error; err != nil {
|
||||
return ctx, errors.Wrap(err, "Casbin 表 ("+i.InitializerName()+") 数据初始化失败!")
|
||||
|
||||
@@ -44,6 +44,12 @@ func SetToken(c *gin.Context, token string, maxAge int) {
|
||||
func GetToken(c *gin.Context) string {
|
||||
// Extend Start: Admin and Gaia JWT
|
||||
token, _ := c.Cookie("x-token")
|
||||
if len(token) == 0 {
|
||||
token = c.Request.Header.Get("Authorization")
|
||||
}
|
||||
if len(token) > 7 && token[0:7] == "Bearer " {
|
||||
token = token[7:]
|
||||
}
|
||||
if token == "" {
|
||||
j := NewJWT()
|
||||
token, _ = c.Cookie("x-token")
|
||||
@@ -65,20 +71,39 @@ func GetClaims(c *gin.Context) (*systemReq.CustomClaims, error) {
|
||||
if err != nil {
|
||||
global.GVA_LOG.Error("从Gin的Context中获取从jwt解析信息失败, 请检查请求头是否存在x-token且claims是否为规定结构")
|
||||
}
|
||||
// 判断是否dify的token
|
||||
if claims.Username == "" {
|
||||
var user system.SysUser
|
||||
var account gaia.Account
|
||||
if err = global.GVA_DB.Where("uuid=?", claims.UserId).First(&user).Error; err == nil {
|
||||
claims.BaseClaims.ID = user.ID
|
||||
claims.Username = user.Username
|
||||
claims.AuthorityId = user.AuthorityId
|
||||
} else if err = global.GVA_DB.Where("id=?", claims.UserId).First(&account).Error; err == nil {
|
||||
if err = global.GVA_DB.Where("email=?", account.Email).First(&user).Error; err == nil {
|
||||
claims.AuthorityId = user.AuthorityId
|
||||
claims.Username = user.Username
|
||||
claims.BaseClaims.ID = user.ID
|
||||
user.UUID = account.ID
|
||||
global.GVA_DB.Save(&user)
|
||||
}
|
||||
}
|
||||
}
|
||||
return claims, err
|
||||
}
|
||||
|
||||
// GetUserID 从Gin的Context中获取从jwt解析出来的用户ID
|
||||
func GetUserID(c *gin.Context) uint {
|
||||
if claims, exists := c.Get("claims"); !exists {
|
||||
if cl, err := GetClaims(c); err != nil {
|
||||
return 0
|
||||
} else {
|
||||
return cl.BaseClaims.ID
|
||||
}
|
||||
} else {
|
||||
if claims, exists := c.Get("claims"); exists {
|
||||
waitUse := claims.(*systemReq.CustomClaims)
|
||||
return waitUse.BaseClaims.ID
|
||||
if waitUse.BaseClaims.ID != 0 {
|
||||
return waitUse.BaseClaims.ID
|
||||
}
|
||||
}
|
||||
if cl, err := GetClaims(c); err != nil {
|
||||
return 0
|
||||
} else {
|
||||
return cl.BaseClaims.ID
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user