Skip to content

Commit

Permalink
add metric (#17125)
Browse files Browse the repository at this point in the history 
1,在resolve的各个环节加监控
2,在create account的各个环节加监控

Approved by: @qingxinhome, @zhangxu19830126, @aptend, @fengttt
  • Loading branch information
@daviszhen
daviszhen authored Jun 25, 2024
1 parent a9b76fa commit dae1706
Show file tree
Hide file tree
Showing 5 changed files with 168 additions and 2 deletions.
37 changes: 37 additions & 0 deletions pkg/frontend/authenticate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ import (
"github.com/matrixorigin/matrixone/pkg/sql/plan/function"
"github.com/matrixorigin/matrixone/pkg/sql/util"
"github.com/matrixorigin/matrixone/pkg/util/metric/mometric"
v2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2"
"github.com/matrixorigin/matrixone/pkg/util/sysview"
"github.com/matrixorigin/matrixone/pkg/util/trace"
"github.com/matrixorigin/matrixone/pkg/util/trace/impl/motrace"
Expand Down Expand Up @@ -3261,6 +3262,10 @@ func getSubscriptionMeta(ctx context.Context, dbName string, ses FeSession, txn
}

func checkSubscriptionValidCommon(ctx context.Context, ses FeSession, subName, accName, pubName string) (subs *plan.SubscriptionMeta, err error) {
start := time.Now()
defer func() {
v2.CheckSubValidDurationHistogram.Observe(time.Since(start).Seconds())
}()
bh := ses.GetBackgroundExec(ctx)
defer bh.Close()
var (
Expand Down Expand Up @@ -7606,6 +7611,10 @@ func InitGeneralTenant(ctx context.Context, ses *Session, ca *createAccount) (er
if !(tenant.IsSysTenant() && tenant.IsMoAdminRole()) {
return moerr.NewInternalError(ctx, "tenant %s user %s role %s do not have the privilege to create the new account", tenant.GetTenant(), tenant.GetUser(), tenant.GetDefaultRole())
}
start := time.Now()
defer func() {
v2.TotalCreateDurationHistogram.Observe(time.Since(start).Seconds())
}()

//normalize the name
err = normalizeNameOfAccount(ctx, ca)
Expand Down Expand Up @@ -7647,6 +7656,7 @@ func InitGeneralTenant(ctx context.Context, ses *Session, ca *createAccount) (er
return rtnErr
}

start1 := time.Now()
//USE the mo_catalog
// MOVE into txn, make sure only create ONE txn.
rtnErr = bh.Exec(ctx, "use mo_catalog;")
Expand All @@ -7672,6 +7682,10 @@ func InitGeneralTenant(ctx context.Context, ses *Session, ca *createAccount) (er
}
}

v2.Step1DurationHistogram.Observe(time.Since(start1).Seconds())

start2 := time.Now()

// create some tables and databases for new account
rtnErr = bh.Exec(newTenantCtx, createMoIndexesSql)
if rtnErr != nil {
Expand Down Expand Up @@ -7707,6 +7721,8 @@ func InitGeneralTenant(ctx context.Context, ses *Session, ca *createAccount) (er
}
}

v2.Step2DurationHistogram.Observe(time.Since(start2).Seconds())

// create tables for new account
rtnErr = createTablesInMoCatalogOfGeneralTenant2(bh, ca, newTenantCtx, newTenant, getGlobalPu())
if rtnErr != nil {
Expand Down Expand Up @@ -7823,10 +7839,17 @@ func createTablesInMoCatalogOfGeneralTenant(ctx context.Context, bh BackgroundEx
}

func createTablesInMoCatalogOfGeneralTenant2(bh BackgroundExec, ca *createAccount, newTenantCtx context.Context, newTenant *TenantInfo, pu *config.ParameterUnit) error {
start := time.Now()
defer func() {
v2.CreateTablesInMoCatalogDurationHistogram.Observe(time.Since(start).Seconds())
}()
var err error
var initDataSqls []string
newTenantCtx, span := trace.Debug(newTenantCtx, "createTablesInMoCatalogOfGeneralTenant2")
defer span.End()

start1 := time.Now()

//create tables for the tenant
for _, sql := range createSqls {
//only the SYS tenant has the table mo_account
Expand All @@ -7839,6 +7862,8 @@ func createTablesInMoCatalogOfGeneralTenant2(bh BackgroundExec, ca *createAccoun
}
}

v2.ExecDDL1DurationHistogram.Observe(time.Since(start1).Seconds())

//initialize the default data of tables for the tenant
addSqlIntoSet := func(sql string) {
initDataSqls = append(initDataSqls, sql)
Expand Down Expand Up @@ -7911,6 +7936,8 @@ func createTablesInMoCatalogOfGeneralTenant2(bh BackgroundExec, ca *createAccoun
addSqlIntoSet(addInitSystemVariablesSql(uint64(newTenant.GetTenantID()), newTenant.GetTenant(), QueryResultMaxsize, pu))
addSqlIntoSet(addInitSystemVariablesSql(uint64(newTenant.GetTenantID()), newTenant.GetTenant(), QueryResultTimeout, pu))

start2 := time.Now()

//fill the mo_role, mo_user, mo_role_privs, mo_user_grant, mo_role_grant
for _, sql := range initDataSqls {
bh.ClearExecResultSet()
Expand All @@ -7919,11 +7946,17 @@ func createTablesInMoCatalogOfGeneralTenant2(bh BackgroundExec, ca *createAccoun
return err
}
}

v2.InitData1DurationHistogram.Observe(time.Since(start2).Seconds())
return nil
}

// createTablesInSystemOfGeneralTenant creates the database system and system_metrics as the external tables.
func createTablesInSystemOfGeneralTenant(ctx context.Context, bh BackgroundExec, newTenant *TenantInfo) error {
start := time.Now()
defer func() {
v2.CreateTablesInSystemDurationHistogram.Observe(time.Since(start).Seconds())
}()
ctx, span := trace.Debug(ctx, "createTablesInSystemOfGeneralTenant")
defer span.End()

Expand All @@ -7948,6 +7981,10 @@ func createTablesInSystemOfGeneralTenant(ctx context.Context, bh BackgroundExec,

// createTablesInInformationSchemaOfGeneralTenant creates the database information_schema and the views or tables.
func createTablesInInformationSchemaOfGeneralTenant(ctx context.Context, bh BackgroundExec) error {
start := time.Now()
defer func() {
v2.CreateTablesInInfoSchemaDurationHistogram.Observe(time.Since(start).Seconds())
}()
ctx, span := trace.Debug(ctx, "createTablesInInformationSchemaOfGeneralTenant")
defer span.End()
//with new tenant
Expand Down
31 changes: 29 additions & 2 deletions pkg/frontend/compiler_context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,11 @@ func (tcc *TxnCompilerContext) getRelation(dbName string, tableName string, sub
return nil, nil, err
}

start := time.Now()
defer func() {
v2.GetRelationDurationHistogram.Observe(time.Since(start).Seconds())
}()

ses := tcc.GetSession()
txn := tcc.GetTxnHandler().GetTxn()
tempCtx := tcc.execCtx.reqCtx
Expand Down Expand Up @@ -291,6 +296,8 @@ func (tcc *TxnCompilerContext) getRelation(dbName string, tableName string, sub
tempCtx = defines.AttachAccountId(tempCtx, uint32(sysAccountID))
}

start1 := time.Now()

//open database
db, err := tcc.GetTxnHandler().GetStorage().Database(tempCtx, dbName, txn)
if err != nil {
Expand All @@ -301,12 +308,16 @@ func (tcc *TxnCompilerContext) getRelation(dbName string, tableName string, sub
return nil, nil, err
}

v2.OpenDBDurationHistogram.Observe(time.Since(start1).Seconds())

// tableNames, err := db.Relations(ctx)
// if err != nil {
// return nil, nil, err
// }
// logDebugf(ses.GetDebugString(), "dbName %v tableNames %v", dbName, tableNames)

start2 := time.Now()

//open table
table, err := db.Relation(tempCtx, tableName, nil)
if err != nil {
Expand All @@ -321,10 +332,17 @@ func (tcc *TxnCompilerContext) getRelation(dbName string, tableName string, sub
table = tmpTable
}
}

v2.OpenTableDurationHistogram.Observe(time.Since(start2).Seconds())

return tempCtx, table, nil
}

func (tcc *TxnCompilerContext) getTmpRelation(ctx context.Context, tableName string) (engine.Relation, error) {
start := time.Now()
defer func() {
v2.GetTmpTableDurationHistogram.Observe(time.Since(start).Seconds())
}()
e := tcc.execCtx.ses.GetTxnHandler().GetStorage()
txn := tcc.execCtx.ses.GetTxnHandler().GetTxn()
db, err := e.Database(ctx, defines.TEMPORARY_DBNAME, txn)
Expand All @@ -339,6 +357,10 @@ func (tcc *TxnCompilerContext) getTmpRelation(ctx context.Context, tableName str
}

func (tcc *TxnCompilerContext) ensureDatabaseIsNotEmpty(dbName string, checkSub bool, snapshot plan2.Snapshot) (string, *plan.SubscriptionMeta, error) {
start := time.Now()
defer func() {
v2.EnsureDatabaseDurationHistogram.Observe(time.Since(start).Seconds())
}()
if len(dbName) == 0 {
dbName = tcc.DefaultDatabase()
}
Expand Down Expand Up @@ -409,7 +431,9 @@ func (tcc *TxnCompilerContext) ResolveSubscriptionTableById(tableId uint64, pubm
func (tcc *TxnCompilerContext) Resolve(dbName string, tableName string, snapshot plan2.Snapshot) (*plan2.ObjectRef, *plan2.TableDef) {
start := time.Now()
defer func() {
v2.TxnStatementResolveDurationHistogram.Observe(time.Since(start).Seconds())
end := time.Since(start).Seconds()
v2.TxnStatementResolveDurationHistogram.Observe(end)
v2.TotalResolveDurationHistogram.Observe(end)
}()

// In order to be compatible with various GUI clients and BI tools, lower case db and table name if it's a mysql system table
Expand Down Expand Up @@ -878,7 +902,10 @@ func (tcc *TxnCompilerContext) GetQueryResultMeta(uuid string) ([]*plan.ColDef,
func (tcc *TxnCompilerContext) GetSubscriptionMeta(dbName string, snapshot plan2.Snapshot) (*plan.SubscriptionMeta, error) {
tempCtx := tcc.execCtx.reqCtx
txn := tcc.GetTxnHandler().GetTxn()

start := time.Now()
defer func() {
v2.GetSubMetaDurationHistogram.Observe(time.Since(start).Seconds())
}()
if plan2.IsSnapshotValid(&snapshot) && snapshot.TS.Less(txn.Txn().SnapshotTS) {
txn = txn.CloneSnapshotOp(*snapshot.TS)

Expand Down
64 changes: 64 additions & 0 deletions pkg/util/metric/v2/dashboard/grafana_dashboard_frontend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ func (c *DashboardCreator) initFrontendDashboard() error {
c.withRowOptions(
c.initFrontendAcceptConnectionDuration(),
c.initFrontendRoutineAndRequestCount(),
c.initFrontendResolveDuration(),
c.initFrontendCreateAccount(),
)...)
if err != nil {
return err
Expand Down Expand Up @@ -100,3 +102,65 @@ func (c *DashboardCreator) initFrontendRoutineAndRequestCount() dashboard.Option
}),
)
}

func (c *DashboardCreator) initFrontendResolveDuration() dashboard.Option {
return dashboard.Row(
"Resolve Duration",
c.getMultiHistogram(
[]string{
c.getMetricWithFilter(`mo_frontend_resolve_duration_bucket`, `label="total-resolve"`),
c.getMetricWithFilter(`mo_frontend_resolve_duration_bucket`, `label="ensure-database"`),
c.getMetricWithFilter(`mo_frontend_resolve_duration_bucket`, `label="get-sub-meta"`),
c.getMetricWithFilter(`mo_frontend_resolve_duration_bucket`, `label="check-sub-valid"`),
c.getMetricWithFilter(`mo_frontend_resolve_duration_bucket`, `label="get-relation"`),
c.getMetricWithFilter(`mo_frontend_resolve_duration_bucket`, `label="open-db"`),
c.getMetricWithFilter(`mo_frontend_resolve_duration_bucket`, `label="open-table"`),
c.getMetricWithFilter(`mo_frontend_resolve_duration_bucket`, `label="get-tmp-table"`),
},
[]string{
"total-resolve",
"ensure-database",
"get-sub-meta",
"check-sub-valid",
"get-relation",
"open-db",
"open-table",
"get-tmp-table",
},
[]float64{0.50, 0.8, 0.90, 0.99},
[]float32{3, 3, 3, 3},
axis.Unit("s"),
axis.Min(0))...,
)
}

func (c *DashboardCreator) initFrontendCreateAccount() dashboard.Option {
return dashboard.Row(
"Create account Duration",
c.getMultiHistogram(
[]string{
c.getMetricWithFilter(`mo_frontend_create_account_duration_bucket`, `label="total-create"`),
c.getMetricWithFilter(`mo_frontend_create_account_duration_bucket`, `label="step1"`),
c.getMetricWithFilter(`mo_frontend_create_account_duration_bucket`, `label="step2"`),
c.getMetricWithFilter(`mo_frontend_create_account_duration_bucket`, `label="create-tables-in-mo-catalog"`),
c.getMetricWithFilter(`mo_frontend_create_account_duration_bucket`, `label="exec-ddl1"`),
c.getMetricWithFilter(`mo_frontend_create_account_duration_bucket`, `label="init-data1"`),
c.getMetricWithFilter(`mo_frontend_create_account_duration_bucket`, `label="create-tables-in-system"`),
c.getMetricWithFilter(`mo_frontend_create_account_duration_bucket`, `label="create-tables-in-info-schema"`),
},
[]string{
"total-create",
"step1",
"step2",
"create-tables-in-mo-catalog",
"exec-ddl1",
"init-data1",
"create-tables-in-system",
"create-tables-in-info-schema",
},
[]float64{0.50, 0.8, 0.90, 0.99},
[]float32{3, 3, 3, 3},
axis.Unit("s"),
axis.Min(0))...,
)
}
36 changes: 36 additions & 0 deletions pkg/util/metric/v2/frontend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,4 +56,40 @@ var (
}, []string{"label"})
StartHandleRequestCounter = requestCounter.WithLabelValues("start-handle")
EndHandleRequestCounter = requestCounter.WithLabelValues("end-handle")

resolveDurationHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "mo",
Subsystem: "frontend",
Name: "resolve_duration",
Help: "Bucketed histogram of txnCompilerContext.Resolve duration.",
Buckets: getDurationBuckets(),
}, []string{"label"})

TotalResolveDurationHistogram = resolveDurationHistogram.WithLabelValues("total-resolve")
EnsureDatabaseDurationHistogram = resolveDurationHistogram.WithLabelValues("ensure-database")
GetSubMetaDurationHistogram = resolveDurationHistogram.WithLabelValues("get-sub-meta")
CheckSubValidDurationHistogram = resolveDurationHistogram.WithLabelValues("check-sub-valid")
GetRelationDurationHistogram = resolveDurationHistogram.WithLabelValues("get-relation")
OpenDBDurationHistogram = resolveDurationHistogram.WithLabelValues("open-db")
OpenTableDurationHistogram = resolveDurationHistogram.WithLabelValues("open-table")
GetTmpTableDurationHistogram = resolveDurationHistogram.WithLabelValues("get-tmp-table")

createAccountDurationHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "mo",
Subsystem: "frontend",
Name: "create_account_duration",
Help: "Bucketed histogram of Create Account duration.",
Buckets: getDurationBuckets(),
}, []string{"label"})

TotalCreateDurationHistogram = createAccountDurationHistogram.WithLabelValues("total-create")
Step1DurationHistogram = createAccountDurationHistogram.WithLabelValues("step1")
Step2DurationHistogram = createAccountDurationHistogram.WithLabelValues("step2")
CreateTablesInMoCatalogDurationHistogram = createAccountDurationHistogram.WithLabelValues("create-tables-in-mo-catalog")
ExecDDL1DurationHistogram = createAccountDurationHistogram.WithLabelValues("exec-ddl1")
InitData1DurationHistogram = createAccountDurationHistogram.WithLabelValues("init-data1")
CreateTablesInSystemDurationHistogram = createAccountDurationHistogram.WithLabelValues("create-tables-in-system")
CreateTablesInInfoSchemaDurationHistogram = createAccountDurationHistogram.WithLabelValues("create-tables-in-info-schema")
)
2 changes: 2 additions & 0 deletions pkg/util/metric/v2/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,8 @@ func initFrontendMetrics() {
registry.MustRegister(acceptConnDurationHistogram)
registry.MustRegister(routineCounter)
registry.MustRegister(requestCounter)
registry.MustRegister(resolveDurationHistogram)
registry.MustRegister(createAccountDurationHistogram)
}

func initPipelineMetrics() {
Expand Down

0 comments on commit dae1706

Please sign in to comment.