Initial submission of data desensitization strategy backend

This commit is contained in:
Liujian
2024-11-21 17:36:13 +08:00
parent 3c59985734
commit ce53bb2d47
26 changed files with 1196 additions and 18 deletions
@@ -0,0 +1,22 @@
package data_masking
type Config struct {
Rules []*Rule `json:"rules"`
}
type Rule struct {
Match *BasicItem `json:"match"`
Mask *Mask `json:"mask"`
}
type BasicItem struct {
Type string `json:"type"`
Value string `json:"value"`
}
type Mask struct {
Type string `json:"type"`
Begin int `json:"begin"`
Length int `json:"length"`
Replace *BasicItem `json:"replace"`
}
@@ -0,0 +1,81 @@
package data_masking
import (
"encoding/json"
"errors"
"fmt"
strategy_driver "github.com/APIParkLab/APIPark/module/strategy/driver"
strategy_dto "github.com/APIParkLab/APIPark/module/strategy/dto"
)
func init() {
strategy_driver.Register(&strategyDriver{confName: "data_mask"})
}
type strategyDriver struct {
confName string
}
func (d *strategyDriver) Driver() string {
return "data-masking"
}
func (d *strategyDriver) ToApinto(s strategy_dto.Strategy) interface{} {
filters := make(map[string][]string)
for _, f := range s.Filters {
filters[f.Name] = f.Values
}
return map[string]interface{}{
"name": s.Filters,
"description": s.Desc,
"priority": s.Priority,
"filters": filters,
d.confName: s.Config,
}
}
func (d *strategyDriver) Check(config interface{}) error {
if config == nil {
return nil
}
data, err := json.Marshal(config)
if err != nil {
return err
}
var cfg Config
err = json.Unmarshal(data, &cfg)
if err != nil {
return err
}
for _, r := range cfg.Rules {
if r.Match == nil {
return errors.New("match can't be null. ")
}
if r.Mask == nil {
return errors.New("mask can't be null. ")
}
if _, ok := validMatchTypes[r.Match.Type]; !ok {
return fmt.Errorf("match type %s is illegal. ", r.Match.Type)
}
if r.Match.Type == "inner" {
if _, ok := validMatchInnerValues[r.Match.Value]; !ok {
return fmt.Errorf("match value %s is illegal. ", r.Match.Value)
}
}
if _, ok := validMaskTypes[r.Mask.Type]; !ok {
return fmt.Errorf("mask type %s is illegal. ", r.Mask.Type)
}
if r.Mask.Replace != nil {
if _, ok := validReplaceTypes[r.Mask.Replace.Type]; !ok {
return fmt.Errorf("replace type %s is illegal. ", r.Mask.Replace.Type)
}
}
}
return nil
}
@@ -0,0 +1,30 @@
package data_masking
var validMatchInnerValues = map[string]struct{}{
"name": {},
"phone": {},
"email": {},
"id-card": {},
"bank-card": {},
"date": {},
"amount": {},
}
var validMatchTypes = map[string]struct{}{
"inner": {},
"keyword": {},
"regex": {},
"json_path": {},
}
var validMaskTypes = map[string]struct{}{
"partial-display": {},
"partial-masking": {},
"truncation": {},
"replacement": {},
"shuffling": {},
}
var validReplaceTypes = map[string]struct{}{
"random": {},
"custom": {},
}
+11
View File
@@ -0,0 +1,11 @@
package strategy_driver
import (
strategy_dto "github.com/APIParkLab/APIPark/module/strategy/dto"
)
type IStrategyDriver interface {
Driver() string
ToApinto(strategy strategy_dto.Strategy) interface{}
Check(config interface{}) error
}
+132
View File
@@ -0,0 +1,132 @@
package strategy_driver
import (
"fmt"
"regexp"
strategy_dto "github.com/APIParkLab/APIPark/module/strategy/dto"
)
type FilterOptionsItem struct {
Name string
Title string
Type string
Pattern *regexp.Regexp
Options []string
}
const (
FilterMethod = "method"
FilterPath = "path"
FilterIP = "ip"
FilterApplication = "application"
FilterApi = "api"
FilterService = "service"
FilterAppKey = "appkey"
FilterTypeRemote = "remote"
FilterTypePattern = "pattern"
FilterTypeStatic = "static"
FilterValuesALL = "ALL"
)
const (
ApiPathRegexp = `^\*?[\w-/]+\*?$`
CIDRIpv4Exp = `^(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([1-9]|[1-2]\d|3[0-2]))?$`
)
const (
HttpALL = "ALL"
HttpGET = "GET"
HttpPOST = "POST"
HttpPUT = "PUT"
HttpDELETE = "DELETE"
HttpPATCH = "PATCH"
HttpHEADER = "HEADER"
HttpOPTIONS = "OPTIONS"
)
var (
staticOptions = []*FilterOptionsItem{
{
Name: FilterMethod,
Title: "API请求方式",
Type: FilterTypeStatic,
Options: []string{HttpALL, HttpGET, HttpPOST, HttpPUT, HttpDELETE, HttpPATCH, HttpHEADER, HttpOPTIONS},
}, {
Name: FilterPath,
Title: "API路径",
Type: FilterTypePattern,
Pattern: regexp.MustCompile(ApiPathRegexp),
}, {
Name: FilterIP,
Title: "IP",
Type: FilterTypePattern,
Pattern: regexp.MustCompile(CIDRIpv4Exp),
},
}
globalFilters = map[string]struct{}{}
serviceFilters = map[string]struct{}{}
)
func init() {
for _, option := range staticOptions {
globalFilters[option.Name] = struct{}{}
}
for _, option := range staticOptions {
serviceFilters[option.Name] = struct{}{}
}
}
func CheckFilters(name string, scope strategy_dto.Scope, filters []*strategy_dto.Filter) error {
var fs map[string]struct{}
switch scope.String() {
case strategy_dto.ScopeSystem:
fs = globalFilters
case strategy_dto.ScopeService:
fs = serviceFilters
default:
return fmt.Errorf("unknown scope %s", scope)
}
filterNameSet := make(map[string]struct{})
for _, filter := range filters {
_, ok := fs[filter.Name]
if !ok {
return fmt.Errorf("%s filter %s not found", name, filter.Name)
}
if len(filter.Values) == 0 {
return fmt.Errorf("%s.Options can't be null. filter.Name:%s ", name, filter.Name)
}
if _, has := filterNameSet[filter.Name]; has {
return fmt.Errorf("%s.Name %s is reduplicative. ", name, filter.Name)
}
filterNameSet[filter.Name] = struct{}{}
}
return nil
}
type OptionTitle struct {
Field string `json:"field"`
Title string `json:"title" aoi18n:""`
}
type FilterOptionConfig struct {
Title string
Titles []OptionTitle
Key string
}
type IFilterOptionHandler interface {
Name() string
Config() FilterOptionConfig
GetOptions(keyword, conditions map[string]interface{}, pageNum, pageSize int) ([]any, int)
Labels(values ...string) []string
Label(value string) string
}
type IFilterFactory interface {
GetHandler(name string) IFilterOptionHandler
}
+52
View File
@@ -0,0 +1,52 @@
package strategy_driver
import (
"fmt"
"github.com/eolinker/eosc"
)
var manager = newManager()
func newManager() *Manager {
return &Manager{
drivers: eosc.BuildUntyped[string, IStrategyDriver](),
}
}
type Manager struct {
drivers eosc.Untyped[string, IStrategyDriver]
}
func (m *Manager) AddDriver(driver IStrategyDriver) {
m.drivers.Set(driver.Driver(), driver)
}
func (m *Manager) GetDriver(driver string) (IStrategyDriver, bool) {
return m.drivers.Get(driver)
}
func (m *Manager) GetDrivers() []string {
return m.drivers.Keys()
}
func (m *Manager) Delete(name string) {
m.drivers.Del(name)
}
func GetDriver(name string) (IStrategyDriver, bool) {
return manager.GetDriver(name)
}
func Register(driver IStrategyDriver) {
manager.AddDriver(driver)
}
func CheckConfig(name string, config interface{}) error {
driver, has := manager.GetDriver(name)
if !has {
return fmt.Errorf("driver %s not found", name)
}
return driver.Check(config)
}
+44
View File
@@ -0,0 +1,44 @@
package strategy_dto
const (
ScopeSystem = "system"
ScopeTeam = "team"
ScopeService = "service"
PublishStatusOnline = "online"
PublishStatusOffline = "offline"
PublishStatusUpdate = "update"
PublishStatusDelete = "delete"
)
type Scope int
func (s Scope) String() string {
switch s {
case 0:
return ScopeSystem
case 1:
return ScopeTeam
case 2:
return ScopeService
default:
return ScopeSystem
}
}
func (s Scope) Int() int {
return int(s)
}
func ToScope(s string) Scope {
switch s {
case ScopeSystem:
return 0
case ScopeTeam:
return 1
case ScopeService:
return 2
default:
return 0
}
}
+29
View File
@@ -0,0 +1,29 @@
package strategy_dto
type Create struct {
Scope Scope `json:"-"`
Target string `json:"-"`
Driver string `json:"-"`
ID string `json:"id"`
Name string `json:"name"`
Priority int `json:"priority"`
Desc string `json:"desc"`
Filters []*Filter `json:"filters"`
Config interface{} `json:"config"`
}
type Edit struct {
Name *string `json:"name"`
Priority *int `json:"priority"`
Desc *string `json:"desc"`
Filters *[]*Filter `json:"filters"`
Config *interface{} `json:"config"`
}
type Filter struct {
Name string `json:"name"`
Values []string `json:"values"`
Type string `json:"type"`
Label string `json:"label"`
Title string `json:"title"`
}
+73
View File
@@ -0,0 +1,73 @@
package strategy_dto
import (
"encoding/json"
"github.com/APIParkLab/APIPark/service/strategy"
"github.com/eolinker/go-common/auto"
)
func ToStrategyItem(s *strategy.Strategy, publishVersion string) *StrategyItem {
publishStatus := PublishStatusOffline
if publishVersion != "" {
if s.IsDelete {
publishStatus = PublishStatusDelete
} else {
version := s.UpdateAt.Format("20060102150405")
if version != publishVersion {
publishStatus = PublishStatusUpdate
} else {
publishStatus = PublishStatusOnline
}
}
}
return &StrategyItem{
Id: s.Id,
Name: s.Name,
Priority: 0,
Desc: s.Desc,
Filters: "",
Updater: auto.UUID(s.Updater),
UpdateTime: auto.TimeLabel(s.UpdateAt),
ProcessedTotal: 0,
PublishStatus: publishStatus,
IsStop: s.IsStop,
}
}
func ToStrategy(s *strategy.Strategy) *Strategy {
filters := make([]*Filter, 0)
json.Unmarshal([]byte(s.Filters), &filters)
var cfg interface{}
json.Unmarshal([]byte(s.Config), &cfg)
return &Strategy{
Id: s.Id,
Name: s.Name,
Priority: s.Priority,
Desc: s.Desc,
Filters: filters,
Config: cfg,
}
}
type Strategy struct {
Id string `json:"id"`
Name string `json:"name"`
Priority int `json:"priority"`
Desc string `json:"desc"`
Filters []*Filter `json:"filters"`
Config interface{} `json:"config"`
}
type StrategyItem struct {
Id string `json:"id"`
Name string `json:"name"`
Priority int `json:"priority"`
Desc string `json:"desc"`
Filters string `json:"filters"`
Updater auto.Label `json:"updater" aolabel:"user"`
UpdateTime auto.TimeLabel `json:"update_time"`
ProcessedTotal int `json:"processed_total"`
PublishStatus string `json:"publish_status"`
IsStop bool `json:"is_stop"`
}
+167
View File
@@ -0,0 +1,167 @@
package strategy
import (
"context"
"encoding/json"
"fmt"
"github.com/eolinker/go-common/store"
"github.com/APIParkLab/APIPark/service/universally/commit"
"github.com/eolinker/go-common/utils"
"github.com/google/uuid"
strategy_driver "github.com/APIParkLab/APIPark/module/strategy/driver"
strategy_dto "github.com/APIParkLab/APIPark/module/strategy/dto"
"github.com/APIParkLab/APIPark/service/strategy"
)
var _ IStrategyModule = (*imlStrategyModule)(nil)
type imlStrategyModule struct {
strategyService strategy.IStrategyService `autowired:""`
transaction store.ITransaction `autowired:""`
}
func (i *imlStrategyModule) Search(ctx context.Context, keyword string, driver string, scope strategy_dto.Scope, target string, page int, pageSize int, filters []string, order ...string) ([]*strategy_dto.StrategyItem, int64, error) {
list, total, err := i.strategyService.Search(ctx, keyword, driver, scope.Int(), target, page, pageSize, filters, order...)
if err != nil {
return nil, 0, err
}
strategyIds := utils.SliceToSlice(list, func(l *strategy.Strategy) string { return l.Id })
commits, err := i.strategyService.ListLatestStrategyCommit(ctx, scope.String(), target, strategyIds...)
if err != nil {
return nil, 0, err
}
commitMap := utils.SliceToMapO(commits, func(c *commit.Commit[strategy.StrategyCommit]) (string, string) { return c.Key, c.Data.Version })
items := make([]*strategy_dto.StrategyItem, 0, len(list))
for _, l := range list {
item := strategy_dto.ToStrategyItem(l, commitMap[l.Id])
items = append(items, item)
}
return items, total, nil
}
func (i *imlStrategyModule) Get(ctx context.Context, id string) (*strategy_dto.Strategy, error) {
info, err := i.strategyService.Get(ctx, id)
if err != nil {
return nil, err
}
return strategy_dto.ToStrategy(info), nil
}
func (i *imlStrategyModule) Create(ctx context.Context, input *strategy_dto.Create) error {
if input.Name == "" {
return fmt.Errorf("name required")
}
if input.ID == "" {
input.ID = uuid.NewString()
}
if input.Priority < 1 {
input.Priority = 1000
}
err := strategy_driver.CheckFilters(input.Driver, input.Scope, input.Filters)
if err != nil {
return err
}
err = strategy_driver.CheckConfig(input.Driver, input.Config)
if err != nil {
return err
}
filters, _ := json.Marshal(input.Filters)
cfg, _ := json.Marshal(input.Config)
return i.strategyService.Create(ctx, &strategy.Create{
Id: input.ID,
Name: input.Name,
Priority: input.Priority,
Desc: input.Desc,
Filters: string(filters),
Config: string(cfg),
Scope: input.Scope.Int(),
Target: input.Target,
Driver: input.Driver,
})
}
func (i *imlStrategyModule) Edit(ctx context.Context, id string, input *strategy_dto.Edit) error {
if input.Name != nil && *input.Name == "" {
return fmt.Errorf("name required")
}
info, err := i.strategyService.Get(ctx, id)
if err != nil {
return err
}
if input.Priority != nil && *input.Priority < 1 {
*input.Priority = 1000
}
filters := info.Filters
if input.Filters != nil {
err = strategy_driver.CheckFilters(info.Driver, strategy_dto.Scope(info.Scope), *input.Filters)
if err != nil {
return err
}
data, _ := json.Marshal(input.Filters)
filters = string(data)
}
cfg := info.Config
if input.Config != nil {
err = strategy_driver.CheckConfig(info.Driver, input.Config)
if err != nil {
return err
}
data, _ := json.Marshal(input.Config)
cfg = string(data)
}
return i.strategyService.Save(ctx, id, &strategy.Edit{
Name: input.Name,
Priority: input.Priority,
Desc: input.Desc,
Filters: &filters,
Config: &cfg,
})
}
func (i *imlStrategyModule) Enable(ctx context.Context, id string) error {
stop := false
return i.strategyService.Save(ctx, id, &strategy.Edit{IsStop: &stop})
}
func (i *imlStrategyModule) Disable(ctx context.Context, id string) error {
stop := true
return i.strategyService.Save(ctx, id, &strategy.Edit{IsStop: &stop})
}
func (i *imlStrategyModule) Publish(ctx context.Context, scope string, target string) error {
list, err := i.strategyService.AllByScope(ctx, strategy_dto.ToScope(scope).Int(), target)
if err != nil {
return err
}
return i.transaction.Transaction(ctx, func(txCtx context.Context) error {
for _, l := range list {
if l.IsDelete {
err = i.strategyService.Delete(ctx, l.Id)
if err != nil {
return err
}
}
// TODO:同步到网关
err = i.strategyService.CommitStrategy(txCtx, scope, target, l.Id, l)
if err != nil {
return err
}
}
return nil
})
}
func (i *imlStrategyModule) Delete(ctx context.Context, id string) error {
return i.strategyService.SortDelete(ctx, id)
}
+29
View File
@@ -0,0 +1,29 @@
package strategy
import (
"context"
"reflect"
"github.com/eolinker/go-common/autowire"
_ "github.com/APIParkLab/APIPark/module/strategy/driver/data-masking"
strategy_dto "github.com/APIParkLab/APIPark/module/strategy/dto"
)
type IStrategyModule interface {
Search(ctx context.Context, keyword string, driver string, scope strategy_dto.Scope, target string, page int, pageSize int, filters []string, order ...string) ([]*strategy_dto.StrategyItem, int64, error)
Get(ctx context.Context, id string) (*strategy_dto.Strategy, error)
Create(ctx context.Context, i *strategy_dto.Create) error
Edit(ctx context.Context, id string, i *strategy_dto.Edit) error
Enable(ctx context.Context, id string) error
Disable(ctx context.Context, id string) error
Publish(ctx context.Context, scope string, target string) error
Delete(ctx context.Context, id string) error
}
func init() {
strategyModule := new(imlStrategyModule)
autowire.Auto[IStrategyModule](func() reflect.Value {
return reflect.ValueOf(strategyModule)
})
}
+2 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"github.com/APIParkLab/APIPark/service/universally/commit"
"github.com/eolinker/go-common/utils"
@@ -34,7 +35,7 @@ type imlUpstreamModule struct {
}
func (i *imlUpstreamModule) ExportAll(ctx context.Context) ([]*upstream_dto.ExportUpstream, error) {
latestCommits, err := i.upstreamService.ListLatestCommit(ctx)
latestCommits, err := i.upstreamService.ListLatestCommit(ctx, cluster.DefaultClusterID)
if err != nil {
return nil, err
}