From bb822801fbf45f79c81a19a01e00774faddc48c7 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Wed, 4 Dec 2024 15:54:09 +0900 Subject: [PATCH 1/7] chore: update index.tsx protocal -> protocol --- .../packages/common/src/components/postcat/api/Icon/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/packages/common/src/components/postcat/api/Icon/index.tsx b/frontend/packages/common/src/components/postcat/api/Icon/index.tsx index 8ab2e0fe..0ecb8709 100644 --- a/frontend/packages/common/src/components/postcat/api/Icon/index.tsx +++ b/frontend/packages/common/src/components/postcat/api/Icon/index.tsx @@ -295,7 +295,7 @@ export interface IconParkIconElement extends HTMLElement { | 'apispace' | 'auto-generate-api' | 'compare-api' - | 'multi-protocal' + | 'multi-protocol' | 'read-good' | 'richdoc' | 'mockapi' From d80bfe61df419dabfb9cb11b6e025698a9bb3ad4 Mon Sep 17 00:00:00 2001 From: Liujian <824010343@qq.com> Date: Fri, 6 Dec 2024 18:48:00 +0800 Subject: [PATCH 2/7] update .gitignore --- .gitignore | 3 +- .gitlab-ci.yml | 98 -------------------------------------------------- 2 files changed, 2 insertions(+), 99 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.gitignore b/.gitignore index f4ef62f4..830ed6b3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ /.idea/ /config.yml /build/ -/apipark \ No newline at end of file +/apipark +.gitlab-ci.yml \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 88137b31..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,98 +0,0 @@ -variables: - PATH: /opt/go-1.21/go/bin/:/opt/node/node/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin - GOROOT: /opt/go-1.21/go - GOPROXY: https://goproxy.cn - VERSION: $CI_COMMIT_SHORT_SHA - APP: apipark - APP_PRE: ${APP}_${VERSION} - BUILD_DIR: ${APP}-build - DEPLOY_DESC: "DEV 环境" - VIEW_ADDR: http://172.18.166.219:8288 - SAVE_DIR: /opt/${APP} - NODE_OPTIONS: --max_old_space_size=8192 - -stages: - - notice - - prefix - - build - - deploy - - webhook - -feishu-informer: # 飞书回调 - stage: notice - variables: - DIFF_URL: "$CI_MERGE_REQUEST_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_IID/diffs" - rules: - - if: $CI_PIPELINE_SOURCE=="merge_request_event" && $CI_COMMIT_BRANCH =~ "main" - script: - - echo "merge request" - - | - curl -X POST -H "Content-Type: application/json" \ - -d "{\"msg_type\":\"text\",\"content\":{\"text\":\"项目：${CI_PROJECT_NAME}\\n提交人：${GITLAB_USER_NAME}\\n提交信息：${CI_MERGE_REQUEST_TITLE}\\n合并分支信息：${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} -> ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}\\n差异性地址：${DIFF_URL}\\n请及时review代码\"}}" \ - https://open.feishu.cn/open-apis/bot/v2/hook/1c334752-2874-41a1-8f1b-3060f2d46b6c - -prebuild: - stage: prefix - rules: - - if: $CI_COMMIT_BRANCH == "main" - script: - - echo "prebuild" - - chmod +x ./scripts/prefix.sh - - ./scripts/prefix.sh - -builder: - stage: build - rules: - - if: $CI_COMMIT_BRANCH == "main" - script: - - set -e - - | - if [ ! -d "../artifacts" ]; then - mkdir -p ../artifacts - fi - if [ -d "../artifacts/dist" ]; then - cp -r ../artifacts/dist frontend/dist - fi - - | - if [ -n "$(git diff --name-status HEAD~1 HEAD -- frontend)" ]; then - ./scripts/build.sh $BUILD_DIR ${VERSION} all "" - else - ./scripts/build.sh $BUILD_DIR ${VERSION} - fi - if [ -d "frontend/dist" ]; then - echo "copy frontend/dist to artifacts/dist" - rm -fr ../artifacts/dist - cp -r frontend/dist ../artifacts/dist - fi - cp $BUILD_DIR/${APP_PRE}_linux_amd64.tar.gz ${SAVE_DIR} - -deployer: - stage: deploy - rules: - - if: $CI_COMMIT_BRANCH == "main" - variables: - APIPARK_GUEST_MODE: allow - APIPARK_GUEST_ID: dklejrfbhjqwdh - script: - - cd ${SAVE_DIR};mkdir -p ${APP_PRE};tar -zxvf ${APP_PRE}_linux_amd64.tar.gz -C ${APP_PRE};cd ${APP_PRE};./install.sh ${SAVE_DIR};./run.sh restart;cd ${SAVE_DIR} && ./clean.sh ${APP_PRE} - when: on_success -success: - stage: webhook - rules: - - if: $CI_COMMIT_BRANCH == "main" - script: - - | - curl -X POST -H "Content-Type: application/json" \ - -d "{\"msg_type\":\"text\",\"content\":{\"text\":\"最近一次提交：${CI_COMMIT_TITLE}\\n提交人：${GITLAB_USER_NAME}\\n项目：${CI_PROJECT_NAME}\\n环境：${DEPLOY_DESC}\\n更新部署完成.\\n访问地址：${VIEW_ADDR}\\n工作流地址：${CI_PIPELINE_URL}\"}}" \ - https://open.feishu.cn/open-apis/bot/v2/hook/c3672932-4dfa-4989-8023-0128bae59338 - when: on_success -failure: - stage: webhook - rules: - - if: $CI_COMMIT_BRANCH == "main" - script: - - | - curl -X POST -H "Content-Type: application/json" \ - -d "{\"msg_type\":\"text\",\"content\":{\"text\":\"最近一次提交：${CI_COMMIT_TITLE}\\n提交人：${GITLAB_USER_NAME}\\n项目：${CI_PROJECT_NAME}\\n环境：${DEPLOY_DESC}\\n更新部署失败，请及时到gitlab上查看\\n工作流地址：${CI_PIPELINE_URL}\"}}" \ - https://open.feishu.cn/open-apis/bot/v2/hook/c3672932-4dfa-4989-8023-0128bae59338 - when: on_failure From bc00f2d57734baf2d3db23bf3253f15b5b5d148e Mon Sep 17 00:00:00 2001 From: Liujian <824010343@qq.com> Date: Mon, 9 Dec 2024 00:17:50 +0800 Subject: [PATCH 3/7] update loki publish --- gateway/apinto/dynamic.go | 1 - gateway/profession.go | 40 ++------------ log-driver/factory.go | 2 +- log-driver/loki/loki.go | 17 +++--- module/log/iml.go | 109 ++++++++++++++++++++++++++++++++++++-- service/log/iml.go | 16 +----- 6 files changed, 121 insertions(+), 64 deletions(-) diff --git a/gateway/apinto/dynamic.go b/gateway/apinto/dynamic.go index d0e891dc..a734a49c 100644 --- a/gateway/apinto/dynamic.go +++ b/gateway/apinto/dynamic.go @@ -20,7 +20,6 @@ func NewDynamicClient(client admin_client.Client, resource string) (*DynamicClie cfg, has := gateway.GetDynamicResourceDriver(resource) if !has { return nil, errors.New("resource not found") - } return &DynamicClient{client: client, profession: cfg.Profession, driver: cfg.Driver}, nil diff --git a/gateway/profession.go b/gateway/profession.go index 3596f388..606a4904 100644 --- a/gateway/profession.go +++ b/gateway/profession.go @@ -57,42 +57,10 @@ var dynamicResourceMap = map[string]Worker{ Profession: ProfessionCertificate, Driver: "server", }, - //"openai": { - // Profession: ProfessionAIProvider, - // Driver: "openai", - //}, - //"google": { - // Profession: ProfessionAIProvider, - // Driver: "google", - //}, - //"anthropic": { - // Profession: ProfessionAIProvider, - // Driver: "anthropic", - //}, - //"moonshot": { - // Profession: ProfessionAIProvider, - // Driver: "moonshot", - //}, - //"tongyi": { - // Profession: ProfessionAIProvider, - // Driver: "tongyi", - //}, - //"zhipuai": { - // Profession: ProfessionAIProvider, - // Driver: "zhipuai", - //}, - //"fireworks": { - // Profession: ProfessionAIProvider, - // Driver: "fireworks", - //}, - //"novita": { - // Profession: ProfessionAIProvider, - // Driver: "novita", - //}, - //"mistralai": { - // Profession: ProfessionAIProvider, - // Driver: "mistralai", - //}, + "loki": { + Profession: ProfessionOutput, + Driver: "loki", + }, } type Worker struct { diff --git a/log-driver/factory.go b/log-driver/factory.go index 49bc3855..ed13eb4d 100644 --- a/log-driver/factory.go +++ b/log-driver/factory.go @@ -7,7 +7,7 @@ var ( ) type IFactory interface { - Create(config string) (ILogDriver, error) + Create(config string) (ILogDriver, map[string]interface{}, error) } type factoryManager struct { diff --git a/log-driver/loki/loki.go b/log-driver/loki/loki.go index 7fe2ddfa..c79cefd4 100644 --- a/log-driver/loki/loki.go +++ b/log-driver/loki/loki.go @@ -21,7 +21,7 @@ func init() { type factory struct { } -func (f *factory) Create(config string) (log_driver.ILogDriver, error) { +func (f *factory) Create(config string) (log_driver.ILogDriver, map[string]interface{}, error) { return NewDriver(config) } @@ -35,24 +35,27 @@ type Driver struct { headers map[string]string } -func NewDriver(config string) (*Driver, error) { +func NewDriver(config string) (*Driver, map[string]interface{}, error) { cfg := new(DriverConfig) err := json.Unmarshal([]byte(config), cfg) if err != nil { - return nil, err + return nil, nil, err } err = cfg.Check() if err != nil { - return nil, err + return nil, nil, err } headers := map[string]string{} for _, h := range cfg.Header { headers[h.Key] = h.Value } return &Driver{ - url: cfg.URL, - headers: headers, - }, nil + url: cfg.URL, + headers: headers, + }, map[string]interface{}{ + "url": cfg.URL, + "headers": headers, + }, nil } func (d *Driver) LogInfo(clusterId string, id string) (*log_driver.LogInfo, error) { diff --git a/module/log/iml.go b/module/log/iml.go index eb3d39d5..ded0a013 100644 --- a/module/log/iml.go +++ b/module/log/iml.go @@ -4,6 +4,13 @@ import ( "context" "encoding/json" "errors" + "time" + + log_driver "github.com/APIParkLab/APIPark/log-driver" + + "github.com/APIParkLab/APIPark/gateway" + + "github.com/eolinker/go-common/store" "gorm.io/gorm" @@ -18,10 +25,58 @@ import ( var _ ILogModule = (*imlLogModule)(nil) type imlLogModule struct { - service log.ILogService `autowired:""` + service log.ILogService `autowired:""` + clusterService cluster.IClusterService `autowired:""` + transaction store.ITransaction `autowired:""` +} + +var labels = map[string]string{ + "cluster": "$cluster", + "node": "$node", +} +var logFormatter = map[string]interface{}{ + "fields": []string{ + "$msec", + "$service", + "$provider", + "$scheme as request_scheme", + "$url as request_uri", + "$host as request_host", + "$header as request_header", + "$remote_addr", + "$request_body", + "$proxy_body", + "$proxy_method", + "$proxy_scheme", + "$proxy_uri", + "$api", + "$proxy_host", + "$proxy_header", + "$proxy_addr", + "$response_headers", + "$status", + "$content_type", + "$proxy_status", + "$request_time", + "$response_time", + "$node", + "$cluster", + "$application", + "$src_ip", + "$block_name as strategy", + "$request_id", + "$request_method", + "$authorization", + "$response_body", + "$proxy_response_body", + }, } func (i *imlLogModule) Save(ctx context.Context, driver string, input *log_dto.Save) error { + factory, has := log_driver.GetFactory(driver) + if !has { + return errors.New("driver not found") + } input.Cluster = cluster.DefaultClusterID var cfg *string if input.Config != nil { @@ -29,10 +84,54 @@ func (i *imlLogModule) Save(ctx context.Context, driver string, input *log_dto.S tmp := string(data) cfg = &tmp } - return i.service.UpdateLogSource(ctx, driver, &log.Save{ - ID: input.ID, - Cluster: &input.Cluster, - Config: cfg, + return i.transaction.Transaction(ctx, func(txCtx context.Context) error { + err := i.service.UpdateLogSource(ctx, driver, &log.Save{ + ID: input.ID, + Cluster: &input.Cluster, + Config: cfg, + }) + if err != nil { + return err + } + info, err := i.service.GetLogSource(ctx, driver) + if err != nil { + return err + } + d, c, err := factory.Create(info.Config) + if err != nil { + return err + } + + client, err := i.clusterService.GatewayClient(ctx, input.Cluster) + if err != nil { + return err + } + dynamicClient, err := client.Dynamic(driver) + if err != nil { + return err + } + attr := make(map[string]interface{}) + attr["driver"] = driver + attr["formatter"] = logFormatter + attr["labels"] = labels + attr["method"] = "POST" + for k, v := range c { + attr[k] = v + } + err = dynamicClient.Online(ctx, &gateway.DynamicRelease{ + BasicItem: &gateway.BasicItem{ + ID: driver, + Description: "collect access log", + Version: time.Now().Format("20060102150405"), + Resource: gateway.ProfessionOutput, + }, + Attr: attr, + }) + if err != nil { + return err + } + log_driver.SetDriver(driver, d) + return nil }) } diff --git a/service/log/iml.go b/service/log/iml.go index 2daeb9a8..e0ff4b56 100644 --- a/service/log/iml.go +++ b/service/log/iml.go @@ -35,7 +35,7 @@ func (i *imlLogService) OnComplete() { if err != nil { continue } - driver, err := factory.Create(s.Config) + driver, _, err := factory.Create(s.Config) if err != nil { continue } @@ -45,10 +45,6 @@ func (i *imlLogService) OnComplete() { } func (i *imlLogService) UpdateLogSource(ctx context.Context, driver string, input *Save) error { - factory, has := log_driver.GetFactory(driver) - if !has { - return errors.New("driver not found") - } s, err := i.store.First(ctx, map[string]interface{}{"driver": driver}) if err != nil { if !errors.Is(err, gorm.ErrRecordNotFound) { @@ -83,15 +79,7 @@ func (i *imlLogService) UpdateLogSource(ctx context.Context, driver string, inpu s.Updater = utils.UserId(ctx) s.UpdateAt = time.Now() } - newDriver, err := factory.Create(s.Config) - if err != nil { - return err - } - err = i.store.Save(ctx, s) - if err != nil { - return err - } - log_driver.SetDriver(driver, newDriver) + return nil } From 3dd44c97e9603090e7f1c2582f1424dd412f22af Mon Sep 17 00:00:00 2001 From: Liujian <824010343@qq.com> Date: Mon, 9 Dec 2024 00:43:05 +0800 Subject: [PATCH 4/7] Automatically publish policies and logs during cluster initialization --- module/log/iml.go | 59 ++++++++++++++++++++++++++++++++++++--- module/log/module.go | 7 ++++- module/strategy/iml.go | 33 ++++++++++++++++++++++ module/strategy/module.go | 3 ++ service/log/iml.go | 21 ++++---------- 5 files changed, 102 insertions(+), 21 deletions(-) diff --git a/module/log/iml.go b/module/log/iml.go index ded0a013..96a8daee 100644 --- a/module/log/iml.go +++ b/module/log/iml.go @@ -85,7 +85,7 @@ func (i *imlLogModule) Save(ctx context.Context, driver string, input *log_dto.S cfg = &tmp } return i.transaction.Transaction(ctx, func(txCtx context.Context) error { - err := i.service.UpdateLogSource(ctx, driver, &log.Save{ + err := i.service.UpdateLogSource(txCtx, driver, &log.Save{ ID: input.ID, Cluster: &input.Cluster, Config: cfg, @@ -93,7 +93,7 @@ func (i *imlLogModule) Save(ctx context.Context, driver string, input *log_dto.S if err != nil { return err } - info, err := i.service.GetLogSource(ctx, driver) + info, err := i.service.GetLogSource(txCtx, driver) if err != nil { return err } @@ -102,10 +102,11 @@ func (i *imlLogModule) Save(ctx context.Context, driver string, input *log_dto.S return err } - client, err := i.clusterService.GatewayClient(ctx, input.Cluster) + client, err := i.clusterService.GatewayClient(txCtx, input.Cluster) if err != nil { return err } + defer client.Close(txCtx) dynamicClient, err := client.Dynamic(driver) if err != nil { return err @@ -118,7 +119,7 @@ func (i *imlLogModule) Save(ctx context.Context, driver string, input *log_dto.S for k, v := range c { attr[k] = v } - err = dynamicClient.Online(ctx, &gateway.DynamicRelease{ + err = dynamicClient.Online(txCtx, &gateway.DynamicRelease{ BasicItem: &gateway.BasicItem{ ID: driver, Description: "collect access log", @@ -159,3 +160,53 @@ func (i *imlLogModule) Get(ctx context.Context, driver string) (*log_dto.LogSour UpdateAt: auto.TimeLabel(info.UpdateAt), }, nil } + +func (i *imlLogModule) initGateway(ctx context.Context, clusterId string, clientDriver gateway.IClientDriver) error { + drivers := log_driver.Drivers() + if len(drivers) < 1 { + return nil + } + + for _, driver := range drivers { + factory, has := log_driver.GetFactory(driver) + if !has { + continue + } + info, err := i.service.GetLogSource(ctx, driver) + if err != nil { + continue + } + d, c, err := factory.Create(info.Config) + if err != nil { + continue + } + + dynamicClient, err := clientDriver.Dynamic(driver) + if err != nil { + continue + } + attr := make(map[string]interface{}) + attr["driver"] = driver + attr["formatter"] = logFormatter + attr["labels"] = labels + attr["method"] = "POST" + for k, v := range c { + attr[k] = v + } + err = dynamicClient.Online(ctx, &gateway.DynamicRelease{ + BasicItem: &gateway.BasicItem{ + ID: driver, + Description: "collect access log", + Version: time.Now().Format("20060102150405"), + Resource: gateway.ProfessionOutput, + }, + Attr: attr, + }) + if err != nil { + continue + } + log_driver.SetDriver(driver, d) + } + + return nil +} diff --git a/module/log/module.go b/module/log/module.go index 548e3db3..bdafdc0c 100644 --- a/module/log/module.go +++ b/module/log/module.go @@ -6,6 +6,7 @@ import ( "github.com/eolinker/go-common/autowire" + "github.com/APIParkLab/APIPark/gateway" log_dto "github.com/APIParkLab/APIPark/module/log/dto" ) @@ -15,7 +16,11 @@ type ILogModule interface { } func init() { + logModule := new(imlLogModule) + autowire.Auto[ILogModule](func() reflect.Value { - return reflect.ValueOf(new(imlLogModule)) + + gateway.RegisterInitHandleFunc(logModule.initGateway) + return reflect.ValueOf(logModule) }) } diff --git a/module/strategy/iml.go b/module/strategy/iml.go index c0b1ec84..b36b9511 100644 --- a/module/strategy/iml.go +++ b/module/strategy/iml.go @@ -359,3 +359,36 @@ func (i *imlStrategyModule) Delete(ctx context.Context, id string) error { } return i.strategyService.SortDelete(ctx, id) } + +func (i *imlStrategyModule) initGateway(ctx context.Context, clusterId string, clientDriver gateway.IClientDriver) error { + commits, err := i.strategyService.ListLatestStrategyCommit(ctx, strategy_dto.ScopeGlobal, "") + if err != nil { + return err + } + publishStrategies := make([]*eosc.Base[gateway.StrategyRelease], 0, len(commits)) + for _, c := range commits { + l := c.Data + if l.IsDelete { + err = i.strategyService.Delete(ctx, l.Id) + if err != nil { + return err + } + } + d, has := strategy_driver.GetDriver(l.Driver) + if !has { + continue + } + publishStrategies = append(publishStrategies, d.ToRelease(strategy_dto.ToStrategy(&strategy.Strategy{ + Id: l.Id, + Name: l.Name, + Priority: l.Priority, + Filters: l.Filters, + Config: l.Config, + Driver: l.Driver, + IsStop: l.IsStop, + IsDelete: l.IsDelete, + }), nil, 5000)) + } + + return clientDriver.Strategy().Online(ctx, publishStrategies...) +} diff --git a/module/strategy/module.go b/module/strategy/module.go index c716f9c0..1dffecae 100644 --- a/module/strategy/module.go +++ b/module/strategy/module.go @@ -5,6 +5,8 @@ import ( "reflect" "time" + "github.com/APIParkLab/APIPark/gateway" + "github.com/eolinker/go-common/autowire" _ "github.com/APIParkLab/APIPark/module/strategy/driver/data-masking" @@ -32,6 +34,7 @@ type IStrategyModule interface { func init() { strategyModule := new(imlStrategyModule) autowire.Auto[IStrategyModule](func() reflect.Value { + gateway.RegisterInitHandleFunc(strategyModule.initGateway) return reflect.ValueOf(strategyModule) }) } diff --git a/service/log/iml.go b/service/log/iml.go index e0ff4b56..9943a15a 100644 --- a/service/log/iml.go +++ b/service/log/iml.go @@ -25,23 +25,7 @@ type imlLogService struct { } func (i *imlLogService) OnComplete() { - drivers := log_driver.Drivers() - for _, d := range drivers { - factory, has := log_driver.GetFactory(d) - if !has { - continue - } - s, err := i.GetLogSource(context.Background(), d) - if err != nil { - continue - } - driver, _, err := factory.Create(s.Config) - if err != nil { - continue - } - log_driver.SetDriver(d, driver) - } } func (i *imlLogService) UpdateLogSource(ctx context.Context, driver string, input *Save) error { @@ -80,6 +64,11 @@ func (i *imlLogService) UpdateLogSource(ctx context.Context, driver string, inpu s.UpdateAt = time.Now() } + err = i.store.Save(ctx, s) + if err != nil { + return err + } + return nil } From 5017fce70cf0516aa2e0d30f5efbd86c6da8f242 Mon Sep 17 00:00:00 2001 From: Liujian <824010343@qq.com> Date: Mon, 9 Dec 2024 21:32:57 +0800 Subject: [PATCH 5/7] update init log --- log-driver/loki/loki.go | 11 +++++++ module/application-authorization/iml.go | 3 ++ module/log/iml.go | 38 +++++++++++++++++++++++-- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/log-driver/loki/loki.go b/log-driver/loki/loki.go index c79cefd4..9f490fbc 100644 --- a/log-driver/loki/loki.go +++ b/log-driver/loki/loki.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/eolinker/eosc/log" + log_driver "github.com/APIParkLab/APIPark/log-driver" ) @@ -69,6 +71,8 @@ func (d *Driver) LogInfo(clusterId string, id string) (*log_driver.LogInfo, erro queries.Set("start", strconv.FormatInt(start.UnixNano(), 10)) queries.Set("end", strconv.FormatInt(now.UnixNano(), 10)) queries.Set("limit", "1") + log.Debug("query is ", queries.Get("query")) + list, err := send[LogInfo](http.MethodGet, fmt.Sprintf("%s/loki/api/v1/query_range", d.url), d.headers, queries, "") if err != nil { return nil, err @@ -103,10 +107,13 @@ func (d *Driver) LogCount(clusterId string, conditions map[string]string, spendH } queries := url.Values{} queries.Set("query", fmt.Sprintf("sum(count_over_time({cluster=\"%s\"} | json %s [%dh])) by (%s)", clusterId, tmpCondition, spendHour, group)) + sendRequestTime := time.Now() list, err := send[LogCount](http.MethodGet, fmt.Sprintf("%s/loki/api/v1/query", d.url), d.headers, queries, "") if err != nil { return nil, err } + log.DebugF("send request spend time: %v", time.Now().Sub(sendRequestTime)) + log.Debug("query is ", queries.Get("query")) result := make(map[string]int64) for _, l := range list { if len(l.Value) != 2 { @@ -161,6 +168,7 @@ func (d *Driver) Logs(clusterId string, conditions map[string]string, start time queries.Set("limit", strconv.FormatInt(limit, 10)) queries.Set("direction", "backward") queries.Set("start", strconv.FormatInt(start.UnixNano(), 10)) + log.Debug("query is ", queries.Get("query")) logs, err := d.recuseLogs(queries, end, offset) if err != nil { return nil, 0, err @@ -262,10 +270,13 @@ func send[T any](method string, uri string, headers map[string]string, queries u for key, value := range headers { req.Header.Set(key, value) } + log.DebugF("do request: %s", uri) + doRequestTime := time.Now() resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } + log.DebugF("do request spend time: %v", time.Now().Sub(doRequestTime)) defer resp.Body.Close() respData, err := io.ReadAll(resp.Body) if err != nil { diff --git a/module/application-authorization/iml.go b/module/application-authorization/iml.go index c872e3f5..9a42ec07 100644 --- a/module/application-authorization/iml.go +++ b/module/application-authorization/iml.go @@ -99,6 +99,9 @@ func (i *imlAuthorizationModule) getApplications(ctx context.Context, appIds []s Expire: a.ExpireTime, Config: authCfg, HideCredential: a.HideCredential, + Label: map[string]string{ + "authorization": a.UUID, + }, } }), } diff --git a/module/log/iml.go b/module/log/iml.go index 96a8daee..6f21b7cb 100644 --- a/module/log/iml.go +++ b/module/log/iml.go @@ -20,6 +20,7 @@ import ( log_dto "github.com/APIParkLab/APIPark/module/log/dto" "github.com/APIParkLab/APIPark/service/log" + log_print "github.com/eolinker/eosc/log" ) var _ ILogModule = (*imlLogModule)(nil) @@ -116,6 +117,8 @@ func (i *imlLogModule) Save(ctx context.Context, driver string, input *log_dto.S attr["formatter"] = logFormatter attr["labels"] = labels attr["method"] = "POST" + attr["scopes"] = []string{"access_log"} + attr["type"] = "json" for k, v := range c { attr[k] = v } @@ -161,6 +164,32 @@ func (i *imlLogModule) Get(ctx context.Context, driver string) (*log_dto.LogSour }, nil } +func (i *imlLogModule) OnComplete() { + drivers := log_driver.Drivers() + if len(drivers) < 1 { + return + } + ctx := context.Background() + for _, driver := range drivers { + factory, has := log_driver.GetFactory(driver) + if !has { + log_print.Errorf("driver %s not found", driver) + continue + } + info, err := i.service.GetLogSource(ctx, driver) + if err != nil { + log_print.Errorf("get log source %s error: %s", driver, err) + continue + } + d, _, err := factory.Create(info.Config) + if err != nil { + log_print.Errorf("create driver %s error: %s,config: %s", driver, err, info.Config) + continue + } + log_driver.SetDriver(driver, d) + } +} + func (i *imlLogModule) initGateway(ctx context.Context, clusterId string, clientDriver gateway.IClientDriver) error { drivers := log_driver.Drivers() if len(drivers) < 1 { @@ -170,19 +199,23 @@ func (i *imlLogModule) initGateway(ctx context.Context, clusterId string, client for _, driver := range drivers { factory, has := log_driver.GetFactory(driver) if !has { + log_print.Errorf("driver %s not found", driver) continue } info, err := i.service.GetLogSource(ctx, driver) if err != nil { + log_print.Errorf("get log source %s error: %s", driver, err) continue } d, c, err := factory.Create(info.Config) if err != nil { + log_print.Errorf("create driver %s error: %s,config: %s", driver, err, info.Config) continue } - + log_driver.SetDriver(driver, d) dynamicClient, err := clientDriver.Dynamic(driver) if err != nil { + log_print.Errorf("get dynamic client %s error: %s", driver, err) continue } attr := make(map[string]interface{}) @@ -203,9 +236,10 @@ func (i *imlLogModule) initGateway(ctx context.Context, clusterId string, client Attr: attr, }) if err != nil { + log_print.Errorf("online driver %s error: %s", driver, err) continue } - log_driver.SetDriver(driver, d) + } return nil From 98304b8816e6d95ee01a4e528f4baad14c648563 Mon Sep 17 00:00:00 2001 From: Liujian <824010343@qq.com> Date: Tue, 10 Dec 2024 17:24:51 +0800 Subject: [PATCH 6/7] fix ai provider upstream bug --- controller/ai-api/iml.go | 12 ++++++------ module/ai/iml.go | 2 +- service/api/iml.go | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/controller/ai-api/iml.go b/controller/ai-api/iml.go index 10543f65..fc59cc5f 100644 --- a/controller/ai-api/iml.go +++ b/controller/ai-api/iml.go @@ -27,7 +27,7 @@ type imlAPIController struct { } func (i *imlAPIController) Create(ctx *gin.Context, serviceId string, input *ai_api_dto.CreateAPI) (*ai_api_dto.API, error) { - info, err := i.serviceModule.Get(ctx, serviceId) + _, err := i.serviceModule.Get(ctx, serviceId) if err != nil { return nil, err } @@ -52,7 +52,7 @@ func (i *imlAPIController) Create(ctx *gin.Context, serviceId string, input *ai_ plugins["ai_formatter"] = api.PluginSetting{ Config: plugin_model.ConfigType{ "model": input.AiModel.Id, - "provider": fmt.Sprintf("%s@ai-provider", info.Provider.Id), + "provider": fmt.Sprintf("%s@ai-provider", input.AiModel.Provider), "config": input.AiModel.Config, }, } @@ -73,7 +73,7 @@ func (i *imlAPIController) Create(ctx *gin.Context, serviceId string, input *ai_ Retry: input.Retry, Plugins: plugins, }, - Upstream: info.Provider.Id, + Upstream: input.AiModel.Provider, Disable: false, }) @@ -86,7 +86,7 @@ func (i *imlAPIController) Create(ctx *gin.Context, serviceId string, input *ai_ } func (i *imlAPIController) Edit(ctx *gin.Context, serviceId string, apiId string, input *ai_api_dto.EditAPI) (*ai_api_dto.API, error) { - info, err := i.serviceModule.Get(ctx, serviceId) + _, err := i.serviceModule.Get(ctx, serviceId) if err != nil { return nil, err } @@ -106,11 +106,11 @@ func (i *imlAPIController) Edit(ctx *gin.Context, serviceId string, apiId string proxy.Plugins["ai_formatter"] = api.PluginSetting{ Config: plugin_model.ConfigType{ "model": input.AiModel.Id, - "provider": fmt.Sprintf("%s@ai-provider", info.Provider.Id), + "provider": fmt.Sprintf("%s@ai-provider", input.AiModel.Provider), "config": input.AiModel.Config, }, } - upstream = &info.Provider.Id + upstream = &input.AiModel.Provider } if input.AiPrompt != nil { diff --git a/module/ai/iml.go b/module/ai/iml.go index 58710ff2..f830a8ac 100644 --- a/module/ai/iml.go +++ b/module/ai/iml.go @@ -34,7 +34,7 @@ func newAIUpstream(provider string, uri model_runtime.IProviderURI) *gateway.Dyn "driver": "http", "balance": "round-robin", "nodes": []string{fmt.Sprintf("%s weight=100", uri.Host())}, - "pass_node": "node", + "pass_host": "node", "scheme": uri.Scheme(), "timeout": 300000, }, diff --git a/service/api/iml.go b/service/api/iml.go index 866bb841..5e81c236 100644 --- a/service/api/iml.go +++ b/service/api/iml.go @@ -162,6 +162,9 @@ func (i *imlAPIService) Save(ctx context.Context, id string, model *Edit) error if model.Disable != nil { ev.Disable = *model.Disable } + if model.Upstream != nil { + ev.Upstream = *model.Upstream + } e := i.apiInfoStore.Save(ctx, ev) if e != nil { From 24c7cd0c8d96585661e66301a9e5d4d2dd6bd0cf Mon Sep 17 00:00:00 2001 From: Liujian <824010343@qq.com> Date: Fri, 13 Dec 2024 18:52:21 +0800 Subject: [PATCH 7/7] fix aksk bug --- .../auth-driver/aksk/aksk.go | 7 +++-- module/log/iml.go | 24 +---------------- service/log/iml.go | 26 ++++++++++++++++++- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/module/application-authorization/auth-driver/aksk/aksk.go b/module/application-authorization/auth-driver/aksk/aksk.go index 083b9ffa..274ae372 100644 --- a/module/application-authorization/auth-driver/aksk/aksk.go +++ b/module/application-authorization/auth-driver/aksk/aksk.go @@ -3,9 +3,9 @@ package aksk import ( "encoding/json" "fmt" - + auth_driver "github.com/APIParkLab/APIPark/module/application-authorization/auth-driver" - + application_authorization_dto "github.com/APIParkLab/APIPark/module/application-authorization/dto" ) @@ -26,8 +26,7 @@ type Config struct { } func (a *Config) ID() string { - //TODO implement me - panic("implement me") + return a.Ak } func (a *Config) Valid() ([]byte, error) { diff --git a/module/log/iml.go b/module/log/iml.go index 6f21b7cb..a6426e7a 100644 --- a/module/log/iml.go +++ b/module/log/iml.go @@ -165,29 +165,7 @@ func (i *imlLogModule) Get(ctx context.Context, driver string) (*log_dto.LogSour } func (i *imlLogModule) OnComplete() { - drivers := log_driver.Drivers() - if len(drivers) < 1 { - return - } - ctx := context.Background() - for _, driver := range drivers { - factory, has := log_driver.GetFactory(driver) - if !has { - log_print.Errorf("driver %s not found", driver) - continue - } - info, err := i.service.GetLogSource(ctx, driver) - if err != nil { - log_print.Errorf("get log source %s error: %s", driver, err) - continue - } - d, _, err := factory.Create(info.Config) - if err != nil { - log_print.Errorf("create driver %s error: %s,config: %s", driver, err, info.Config) - continue - } - log_driver.SetDriver(driver, d) - } + } func (i *imlLogModule) initGateway(ctx context.Context, clusterId string, clientDriver gateway.IClientDriver) error { diff --git a/service/log/iml.go b/service/log/iml.go index 9943a15a..f514dc31 100644 --- a/service/log/iml.go +++ b/service/log/iml.go @@ -5,6 +5,8 @@ import ( "errors" "time" + log_print "github.com/eolinker/eosc/log" + "github.com/google/uuid" log_driver "github.com/APIParkLab/APIPark/log-driver" @@ -25,7 +27,29 @@ type imlLogService struct { } func (i *imlLogService) OnComplete() { - + drivers := log_driver.Drivers() + if len(drivers) < 1 { + return + } + ctx := context.Background() + for _, driver := range drivers { + factory, has := log_driver.GetFactory(driver) + if !has { + log_print.Errorf("driver %s not found", driver) + continue + } + info, err := i.GetLogSource(ctx, driver) + if err != nil { + log_print.Errorf("get log source %s error: %s", driver, err) + continue + } + d, _, err := factory.Create(info.Config) + if err != nil { + log_print.Errorf("create driver %s error: %s,config: %s", driver, err, info.Config) + continue + } + log_driver.SetDriver(driver, d) + } } func (i *imlLogService) UpdateLogSource(ctx context.Context, driver string, input *Save) error {