Merge branch 'main' of github.com:APIParkLab/APIPark

.gitignore

@@ -2,4 +2,5 @@
 /.idea/
 /config.yml
 /build/
-/apipark
+/apipark
+.gitlab-ci.yml

.gitlab-ci.yml

@@ -1,98 +0,0 @@
-variables:
-  PATH: /opt/go-1.21/go/bin/:/opt/node/node/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
-  GOROOT: /opt/go-1.21/go
-  GOPROXY: https://goproxy.cn
-  VERSION: $CI_COMMIT_SHORT_SHA
-  APP: apipark
-  APP_PRE: ${APP}_${VERSION}
-  BUILD_DIR: ${APP}-build
-  DEPLOY_DESC: "DEV 环境"
-  VIEW_ADDR: http://172.18.166.219:8288
-  SAVE_DIR: /opt/${APP}
-  NODE_OPTIONS: --max_old_space_size=8192
-
-stages:
-  - notice
-  - prefix
-  - build
-  - deploy
-  - webhook
-
-feishu-informer: # 飞书回调
-  stage: notice
-  variables:
-    DIFF_URL: "$CI_MERGE_REQUEST_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_IID/diffs"
-  rules:
-    - if: $CI_PIPELINE_SOURCE=="merge_request_event" && $CI_COMMIT_BRANCH =~ "main"
-  script:
-    - echo "merge request"
-    - |
-      curl -X POST -H "Content-Type: application/json" \
-      -d "{\"msg_type\":\"text\",\"content\":{\"text\":\"项目：${CI_PROJECT_NAME}\\n提交人：${GITLAB_USER_NAME}\\n提交信息：${CI_MERGE_REQUEST_TITLE}\\n合并分支信息：${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} -> ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}\\n差异性地址：${DIFF_URL}\\n请及时review代码\"}}" \
-      https://open.feishu.cn/open-apis/bot/v2/hook/1c334752-2874-41a1-8f1b-3060f2d46b6c
-
-prebuild:
-  stage: prefix
-  rules:
-    - if:  $CI_COMMIT_BRANCH == "main"
-  script:
-    - echo "prebuild"
-    - chmod +x ./scripts/prefix.sh
-    - ./scripts/prefix.sh
-
-builder:
-  stage: build
-  rules:
-    - if: $CI_COMMIT_BRANCH == "main"
-  script:
-    - set -e
-    - |
-      if [ ! -d "../artifacts" ]; then
-        mkdir -p ../artifacts
-      fi
-      if [ -d "../artifacts/dist" ]; then
-        cp -r ../artifacts/dist frontend/dist
-      fi
-    - |
-      if [ -n "$(git diff --name-status HEAD~1 HEAD -- frontend)" ]; then
-        ./scripts/build.sh $BUILD_DIR ${VERSION} all ""
-      else
-        ./scripts/build.sh $BUILD_DIR ${VERSION}
-      fi
-      if [ -d "frontend/dist" ]; then
-        echo "copy frontend/dist to artifacts/dist"
-        rm -fr ../artifacts/dist
-        cp -r frontend/dist ../artifacts/dist
-      fi
-      cp $BUILD_DIR/${APP_PRE}_linux_amd64.tar.gz ${SAVE_DIR}
-
-deployer:
-  stage: deploy
-  rules:
-    - if: $CI_COMMIT_BRANCH == "main"
-  variables:
-    APIPARK_GUEST_MODE: allow
-    APIPARK_GUEST_ID: dklejrfbhjqwdh
-  script:
-    - cd ${SAVE_DIR};mkdir -p ${APP_PRE};tar -zxvf ${APP_PRE}_linux_amd64.tar.gz -C ${APP_PRE};cd ${APP_PRE};./install.sh ${SAVE_DIR};./run.sh restart;cd ${SAVE_DIR} && ./clean.sh ${APP_PRE}
-  when: on_success
-success:
-  stage: webhook
-  rules:
-    - if: $CI_COMMIT_BRANCH == "main"
-  script:
-    - |
-      curl -X POST -H "Content-Type: application/json" \
-      -d "{\"msg_type\":\"text\",\"content\":{\"text\":\"最近一次提交：${CI_COMMIT_TITLE}\\n提交人：${GITLAB_USER_NAME}\\n项目：${CI_PROJECT_NAME}\\n环境：${DEPLOY_DESC}\\n更新部署完成.\\n访问地址：${VIEW_ADDR}\\n工作流地址：${CI_PIPELINE_URL}\"}}" \
-      https://open.feishu.cn/open-apis/bot/v2/hook/c3672932-4dfa-4989-8023-0128bae59338
-  when: on_success
-failure:
-  stage: webhook
-  rules:
-    - if: $CI_COMMIT_BRANCH == "main"
-  script:
-    - |
-      curl -X POST -H "Content-Type: application/json" \
-      -d "{\"msg_type\":\"text\",\"content\":{\"text\":\"最近一次提交：${CI_COMMIT_TITLE}\\n提交人：${GITLAB_USER_NAME}\\n项目：${CI_PROJECT_NAME}\\n环境：${DEPLOY_DESC}\\n更新部署失败，请及时到gitlab上查看\\n工作流地址：${CI_PIPELINE_URL}\"}}" \
-      https://open.feishu.cn/open-apis/bot/v2/hook/c3672932-4dfa-4989-8023-0128bae59338
-  when: on_failure

frontend/packages/common/src/components/postcat/api/Icon/index.tsx

@@ -295,7 +295,7 @@ export interface IconParkIconElement extends HTMLElement {
       | 'apispace'
       | 'auto-generate-api'
       | 'compare-api'
-      | 'multi-protocal'
+      | 'multi-protocol'
       | 'read-good'
       | 'richdoc'
       | 'mockapi'

gateway/apinto/dynamic.go

@@ -20,7 +20,6 @@ func NewDynamicClient(client admin_client.Client, resource string) (*DynamicClie
 	cfg, has := gateway.GetDynamicResourceDriver(resource)
 	if !has {
 		return nil, errors.New("resource not found")
-
 	}
 
 	return &DynamicClient{client: client, profession: cfg.Profession, driver: cfg.Driver}, nil

gateway/profession.go

@@ -57,42 +57,10 @@ var dynamicResourceMap = map[string]Worker{
 		Profession: ProfessionCertificate,
 		Driver:     "server",
 	},
-	//"openai": {
-	//	Profession: ProfessionAIProvider,
-	//	Driver:     "openai",
-	//},
-	//"google": {
-	//	Profession: ProfessionAIProvider,
-	//	Driver:     "google",
-	//},
-	//"anthropic": {
-	//	Profession: ProfessionAIProvider,
-	//	Driver:     "anthropic",
-	//},
-	//"moonshot": {
-	//	Profession: ProfessionAIProvider,
-	//	Driver:     "moonshot",
-	//},
-	//"tongyi": {
-	//	Profession: ProfessionAIProvider,
-	//	Driver:     "tongyi",
-	//},
-	//"zhipuai": {
-	//	Profession: ProfessionAIProvider,
-	//	Driver:     "zhipuai",
-	//},
-	//"fireworks": {
-	//	Profession: ProfessionAIProvider,
-	//	Driver:     "fireworks",
-	//},
-	//"novita": {
-	//	Profession: ProfessionAIProvider,
-	//	Driver:     "novita",
-	//},
-	//"mistralai": {
-	//	Profession: ProfessionAIProvider,
-	//	Driver:     "mistralai",
-	//},
+	"loki": {
+		Profession: ProfessionOutput,
+		Driver:     "loki",
+	},
 }
 
 type Worker struct {

log-driver/factory.go

@@ -7,7 +7,7 @@ var (
 )
 
 type IFactory interface {
-	Create(config string) (ILogDriver, error)
+	Create(config string) (ILogDriver, map[string]interface{}, error)
 }
 
 type factoryManager struct {

log-driver/loki/loki.go

@@ -11,6 +11,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/eolinker/eosc/log"
+
 	log_driver "github.com/APIParkLab/APIPark/log-driver"
 )
 
@@ -21,7 +23,7 @@ func init() {
 type factory struct {
 }
 
-func (f *factory) Create(config string) (log_driver.ILogDriver, error) {
+func (f *factory) Create(config string) (log_driver.ILogDriver, map[string]interface{}, error) {
 
 	return NewDriver(config)
 }
@@ -35,24 +37,27 @@ type Driver struct {
 	headers map[string]string
 }
 
-func NewDriver(config string) (*Driver, error) {
+func NewDriver(config string) (*Driver, map[string]interface{}, error) {
 	cfg := new(DriverConfig)
 	err := json.Unmarshal([]byte(config), cfg)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	err = cfg.Check()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	headers := map[string]string{}
 	for _, h := range cfg.Header {
 		headers[h.Key] = h.Value
 	}
 	return &Driver{
-		url:     cfg.URL,
-		headers: headers,
-	}, nil
+			url:     cfg.URL,
+			headers: headers,
+		}, map[string]interface{}{
+			"url":     cfg.URL,
+			"headers": headers,
+		}, nil
 }
 
 func (d *Driver) LogInfo(clusterId string, id string) (*log_driver.LogInfo, error) {
@@ -66,6 +71,8 @@ func (d *Driver) LogInfo(clusterId string, id string) (*log_driver.LogInfo, erro
 	queries.Set("start", strconv.FormatInt(start.UnixNano(), 10))
 	queries.Set("end", strconv.FormatInt(now.UnixNano(), 10))
 	queries.Set("limit", "1")
+	log.Debug("query is ", queries.Get("query"))
+
 	list, err := send[LogInfo](http.MethodGet, fmt.Sprintf("%s/loki/api/v1/query_range", d.url), d.headers, queries, "")
 	if err != nil {
 		return nil, err
@@ -100,10 +107,13 @@ func (d *Driver) LogCount(clusterId string, conditions map[string]string, spendH
 	}
 	queries := url.Values{}
 	queries.Set("query", fmt.Sprintf("sum(count_over_time({cluster=\"%s\"} | json %s [%dh])) by (%s)", clusterId, tmpCondition, spendHour, group))
+	sendRequestTime := time.Now()
 	list, err := send[LogCount](http.MethodGet, fmt.Sprintf("%s/loki/api/v1/query", d.url), d.headers, queries, "")
 	if err != nil {
 		return nil, err
 	}
+	log.DebugF("send request spend time: %v", time.Now().Sub(sendRequestTime))
+	log.Debug("query is ", queries.Get("query"))
 	result := make(map[string]int64)
 	for _, l := range list {
 		if len(l.Value) != 2 {
@@ -158,6 +168,7 @@ func (d *Driver) Logs(clusterId string, conditions map[string]string, start time
 	queries.Set("limit", strconv.FormatInt(limit, 10))
 	queries.Set("direction", "backward")
 	queries.Set("start", strconv.FormatInt(start.UnixNano(), 10))
+	log.Debug("query is ", queries.Get("query"))
 	logs, err := d.recuseLogs(queries, end, offset)
 	if err != nil {
 		return nil, 0, err
@@ -259,10 +270,13 @@ func send[T any](method string, uri string, headers map[string]string, queries u
 	for key, value := range headers {
 		req.Header.Set(key, value)
 	}
+	log.DebugF("do request: %s", uri)
+	doRequestTime := time.Now()
 	resp, err := client.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("failed to send request: %w", err)
 	}
+	log.DebugF("do request spend time: %v", time.Now().Sub(doRequestTime))
 	defer resp.Body.Close()
 	respData, err := io.ReadAll(resp.Body)
 	if err != nil {

module/application-authorization/iml.go

@@ -99,6 +99,9 @@ func (i *imlAuthorizationModule) getApplications(ctx context.Context, appIds []s
 					Expire:         a.ExpireTime,
 					Config:         authCfg,
 					HideCredential: a.HideCredential,
+					Label: map[string]string{
+						"authorization": a.UUID,
+					},
 				}
 			}),
 		}

module/log/iml.go

@@ -4,6 +4,13 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"time"
+
+	log_driver "github.com/APIParkLab/APIPark/log-driver"
+
+	"github.com/APIParkLab/APIPark/gateway"
+
+	"github.com/eolinker/go-common/store"
 
 	"gorm.io/gorm"
 
@@ -13,15 +20,64 @@ import (
 
 	log_dto "github.com/APIParkLab/APIPark/module/log/dto"
 	"github.com/APIParkLab/APIPark/service/log"
+	log_print "github.com/eolinker/eosc/log"
 )
 
 var _ ILogModule = (*imlLogModule)(nil)
 
 type imlLogModule struct {
-	service log.ILogService `autowired:""`
+	service        log.ILogService         `autowired:""`
+	clusterService cluster.IClusterService `autowired:""`
+	transaction    store.ITransaction      `autowired:""`
+}
+
+var labels = map[string]string{
+	"cluster": "$cluster",
+	"node":    "$node",
+}
+var logFormatter = map[string]interface{}{
+	"fields": []string{
+		"$msec",
+		"$service",
+		"$provider",
+		"$scheme as request_scheme",
+		"$url as request_uri",
+		"$host as request_host",
+		"$header as request_header",
+		"$remote_addr",
+		"$request_body",
+		"$proxy_body",
+		"$proxy_method",
+		"$proxy_scheme",
+		"$proxy_uri",
+		"$api",
+		"$proxy_host",
+		"$proxy_header",
+		"$proxy_addr",
+		"$response_headers",
+		"$status",
+		"$content_type",
+		"$proxy_status",
+		"$request_time",
+		"$response_time",
+		"$node",
+		"$cluster",
+		"$application",
+		"$src_ip",
+		"$block_name as strategy",
+		"$request_id",
+		"$request_method",
+		"$authorization",
+		"$response_body",
+		"$proxy_response_body",
+	},
 }
 
 func (i *imlLogModule) Save(ctx context.Context, driver string, input *log_dto.Save) error {
+	factory, has := log_driver.GetFactory(driver)
+	if !has {
+		return errors.New("driver not found")
+	}
 	input.Cluster = cluster.DefaultClusterID
 	var cfg *string
 	if input.Config != nil {
@@ -29,10 +85,57 @@ func (i *imlLogModule) Save(ctx context.Context, driver string, input *log_dto.S
 		tmp := string(data)
 		cfg = &tmp
 	}
-	return i.service.UpdateLogSource(ctx, driver, &log.Save{
-		ID:      input.ID,
-		Cluster: &input.Cluster,
-		Config:  cfg,
+	return i.transaction.Transaction(ctx, func(txCtx context.Context) error {
+		err := i.service.UpdateLogSource(txCtx, driver, &log.Save{
+			ID:      input.ID,
+			Cluster: &input.Cluster,
+			Config:  cfg,
+		})
+		if err != nil {
+			return err
+		}
+		info, err := i.service.GetLogSource(txCtx, driver)
+		if err != nil {
+			return err
+		}
+		d, c, err := factory.Create(info.Config)
+		if err != nil {
+			return err
+		}
+
+		client, err := i.clusterService.GatewayClient(txCtx, input.Cluster)
+		if err != nil {
+			return err
+		}
+		defer client.Close(txCtx)
+		dynamicClient, err := client.Dynamic(driver)
+		if err != nil {
+			return err
+		}
+		attr := make(map[string]interface{})
+		attr["driver"] = driver
+		attr["formatter"] = logFormatter
+		attr["labels"] = labels
+		attr["method"] = "POST"
+		attr["scopes"] = []string{"access_log"}
+		attr["type"] = "json"
+		for k, v := range c {
+			attr[k] = v
+		}
+		err = dynamicClient.Online(txCtx, &gateway.DynamicRelease{
+			BasicItem: &gateway.BasicItem{
+				ID:          driver,
+				Description: "collect access log",
+				Version:     time.Now().Format("20060102150405"),
+				Resource:    gateway.ProfessionOutput,
+			},
+			Attr: attr,
+		})
+		if err != nil {
+			return err
+		}
+		log_driver.SetDriver(driver, d)
+		return nil
 	})
 }
 
@@ -60,3 +163,84 @@ func (i *imlLogModule) Get(ctx context.Context, driver string) (*log_dto.LogSour
 		UpdateAt: auto.TimeLabel(info.UpdateAt),
 	}, nil
 }
+
+func (i *imlLogModule) OnComplete() {
+	drivers := log_driver.Drivers()
+	if len(drivers) < 1 {
+		return
+	}
+	ctx := context.Background()
+	for _, driver := range drivers {
+		factory, has := log_driver.GetFactory(driver)
+		if !has {
+			log_print.Errorf("driver %s not found", driver)
+			continue
+		}
+		info, err := i.service.GetLogSource(ctx, driver)
+		if err != nil {
+			log_print.Errorf("get log source %s error: %s", driver, err)
+			continue
+		}
+		d, _, err := factory.Create(info.Config)
+		if err != nil {
+			log_print.Errorf("create driver %s error: %s,config: %s", driver, err, info.Config)
+			continue
+		}
+		log_driver.SetDriver(driver, d)
+	}
+}
+
+func (i *imlLogModule) initGateway(ctx context.Context, clusterId string, clientDriver gateway.IClientDriver) error {
+	drivers := log_driver.Drivers()
+	if len(drivers) < 1 {
+		return nil
+	}
+
+	for _, driver := range drivers {
+		factory, has := log_driver.GetFactory(driver)
+		if !has {
+			log_print.Errorf("driver %s not found", driver)
+			continue
+		}
+		info, err := i.service.GetLogSource(ctx, driver)
+		if err != nil {
+			log_print.Errorf("get log source %s error: %s", driver, err)
+			continue
+		}
+		d, c, err := factory.Create(info.Config)
+		if err != nil {
+			log_print.Errorf("create driver %s error: %s,config: %s", driver, err, info.Config)
+			continue
+		}
+		log_driver.SetDriver(driver, d)
+		dynamicClient, err := clientDriver.Dynamic(driver)
+		if err != nil {
+			log_print.Errorf("get dynamic client %s error: %s", driver, err)
+			continue
+		}
+		attr := make(map[string]interface{})
+		attr["driver"] = driver
+		attr["formatter"] = logFormatter
+		attr["labels"] = labels
+		attr["method"] = "POST"
+		for k, v := range c {
+			attr[k] = v
+		}
+		err = dynamicClient.Online(ctx, &gateway.DynamicRelease{
+			BasicItem: &gateway.BasicItem{
+				ID:          driver,
+				Description: "collect access log",
+				Version:     time.Now().Format("20060102150405"),
+				Resource:    gateway.ProfessionOutput,
+			},
+			Attr: attr,
+		})
+		if err != nil {
+			log_print.Errorf("online driver %s error: %s", driver, err)
+			continue
+		}
+
+	}
+
+	return nil
+}

module/log/module.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/eolinker/go-common/autowire"
 
+	"github.com/APIParkLab/APIPark/gateway"
 	log_dto "github.com/APIParkLab/APIPark/module/log/dto"
 )
 
@@ -15,7 +16,11 @@ type ILogModule interface {
 }
 
 func init() {
+	logModule := new(imlLogModule)
+
 	autowire.Auto[ILogModule](func() reflect.Value {
-		return reflect.ValueOf(new(imlLogModule))
+
+		gateway.RegisterInitHandleFunc(logModule.initGateway)
+		return reflect.ValueOf(logModule)
 	})
 }

module/strategy/iml.go

@@ -359,3 +359,36 @@ func (i *imlStrategyModule) Delete(ctx context.Context, id string) error {
 	}
 	return i.strategyService.SortDelete(ctx, id)
 }
+
+func (i *imlStrategyModule) initGateway(ctx context.Context, clusterId string, clientDriver gateway.IClientDriver) error {
+	commits, err := i.strategyService.ListLatestStrategyCommit(ctx, strategy_dto.ScopeGlobal, "")
+	if err != nil {
+		return err
+	}
+	publishStrategies := make([]*eosc.Base[gateway.StrategyRelease], 0, len(commits))
+	for _, c := range commits {
+		l := c.Data
+		if l.IsDelete {
+			err = i.strategyService.Delete(ctx, l.Id)
+			if err != nil {
+				return err
+			}
+		}
+		d, has := strategy_driver.GetDriver(l.Driver)
+		if !has {
+			continue
+		}
+		publishStrategies = append(publishStrategies, d.ToRelease(strategy_dto.ToStrategy(&strategy.Strategy{
+			Id:       l.Id,
+			Name:     l.Name,
+			Priority: l.Priority,
+			Filters:  l.Filters,
+			Config:   l.Config,
+			Driver:   l.Driver,
+			IsStop:   l.IsStop,
+			IsDelete: l.IsDelete,
+		}), nil, 5000))
+	}
+
+	return clientDriver.Strategy().Online(ctx, publishStrategies...)
+}

module/strategy/module.go

@@ -5,6 +5,8 @@ import (
 	"reflect"
 	"time"
 
+	"github.com/APIParkLab/APIPark/gateway"
+
 	"github.com/eolinker/go-common/autowire"
 
 	_ "github.com/APIParkLab/APIPark/module/strategy/driver/data-masking"
@@ -32,6 +34,7 @@ type IStrategyModule interface {
 func init() {
 	strategyModule := new(imlStrategyModule)
 	autowire.Auto[IStrategyModule](func() reflect.Value {
+		gateway.RegisterInitHandleFunc(strategyModule.initGateway)
 		return reflect.ValueOf(strategyModule)
 	})
 }

service/log/iml.go

@@ -25,30 +25,10 @@ type imlLogService struct {
 }
 
 func (i *imlLogService) OnComplete() {
-	drivers := log_driver.Drivers()
-	for _, d := range drivers {
-		factory, has := log_driver.GetFactory(d)
-		if !has {
-			continue
-		}
-		s, err := i.GetLogSource(context.Background(), d)
-		if err != nil {
-			continue
-		}
-		driver, err := factory.Create(s.Config)
-		if err != nil {
-			continue
-		}
-		log_driver.SetDriver(d, driver)
 
-	}
 }
 
 func (i *imlLogService) UpdateLogSource(ctx context.Context, driver string, input *Save) error {
-	factory, has := log_driver.GetFactory(driver)
-	if !has {
-		return errors.New("driver not found")
-	}
 	s, err := i.store.First(ctx, map[string]interface{}{"driver": driver})
 	if err != nil {
 		if !errors.Is(err, gorm.ErrRecordNotFound) {
@@ -83,15 +63,12 @@ func (i *imlLogService) UpdateLogSource(ctx context.Context, driver string, inpu
 		s.Updater = utils.UserId(ctx)
 		s.UpdateAt = time.Now()
 	}
-	newDriver, err := factory.Create(s.Config)
-	if err != nil {
-		return err
-	}
+
 	err = i.store.Save(ctx, s)
 	if err != nil {
 		return err
 	}
-	log_driver.SetDriver(driver, newDriver)
+
 	return nil
 }