diff --git a/pkg/domain/domain.go b/pkg/domain/domain.go index b99970a6be966..fca15810fb7df 100644 --- a/pkg/domain/domain.go +++ b/pkg/domain/domain.go @@ -3182,7 +3182,7 @@ func (do *Domain) planCacheMetricsAndVars() { // planCacheEvictTrigger triggers the plan cache eviction periodically. func (do *Domain) planCacheEvictTrigger() { defer util.Recover(metrics.LabelDomain, "planCacheEvictTrigger", nil, false) - ticker := time.NewTicker(time.Second * 15) // 15s by default + ticker := time.NewTicker(time.Second * 30) // 30s by default defer func() { ticker.Stop() logutil.BgLogger().Info("planCacheEvictTrigger exited.") @@ -3192,7 +3192,13 @@ func (do *Domain) planCacheEvictTrigger() { select { case <-ticker.C: // trigger the eviction - do.instancePlanCache.Evict() + begin := time.Now() + detailInfo, numEvicted := do.instancePlanCache.Evict() + metrics2.GetPlanCacheInstanceEvict().Set(float64(numEvicted)) + logutil.BgLogger().Info("instance plan eviction", + zap.String("detail", detailInfo), + zap.Int64("num_evicted", int64(numEvicted)), + zap.Duration("time_spent", time.Since(begin))) case <-do.exit: return } diff --git a/pkg/planner/core/metrics/metrics.go b/pkg/planner/core/metrics/metrics.go index 9213a5df381b5..4576ca67721a9 100644 --- a/pkg/planner/core/metrics/metrics.go +++ b/pkg/planner/core/metrics/metrics.go @@ -32,6 +32,7 @@ var ( sessionPlanCacheInstanceMemoryUsage prometheus.Gauge instancePlanCacheInstancePlanNumCounter prometheus.Gauge instancePlanCacheInstanceMemoryUsage prometheus.Gauge + instancePlanCacheInstanceNumEvict prometheus.Gauge ) func init() { @@ -52,6 +53,7 @@ func InitMetricsVars() { sessionPlanCacheInstanceMemoryUsage = metrics.PlanCacheInstanceMemoryUsage.WithLabelValues(" session-plan-cache") instancePlanCacheInstancePlanNumCounter = metrics.PlanCacheInstancePlanNumCounter.WithLabelValues(" instance-plan-cache") instancePlanCacheInstanceMemoryUsage = metrics.PlanCacheInstanceMemoryUsage.WithLabelValues(" instance-plan-cache") + instancePlanCacheInstanceNumEvict = metrics.PlanCacheInstancePlanNumCounter.WithLabelValues(" instance-plan-cache-last-evict") } // GetPlanCacheHitCounter get different plan cache hit counter @@ -90,3 +92,8 @@ func GetPlanCacheInstanceMemoryUsage(instancePlanCache bool) prometheus.Gauge { } return sessionPlanCacheInstanceMemoryUsage } + +// GetPlanCacheInstanceEvict get instance plan cache evict counter. +func GetPlanCacheInstanceEvict() prometheus.Gauge { + return instancePlanCacheInstanceNumEvict +} diff --git a/pkg/planner/core/plan_cache_instance.go b/pkg/planner/core/plan_cache_instance.go index 105f32d63f55d..5cdfbab6a8de1 100644 --- a/pkg/planner/core/plan_cache_instance.go +++ b/pkg/planner/core/plan_cache_instance.go @@ -15,6 +15,7 @@ package core import ( + "fmt" "sort" "sync" "time" @@ -55,6 +56,7 @@ type instancePlanCache struct { totPlan atomic.Int64 evictMutex sync.Mutex + inEvict atomic.Bool softMemLimit atomic.Int64 hardMemLimit atomic.Int64 } @@ -84,10 +86,12 @@ func (pc *instancePlanCache) Get(key string, paramTypes any) (value any, ok bool return pc.getPlanFromList(headNode, paramTypes) } -func (*instancePlanCache) getPlanFromList(headNode *instancePCNode, paramTypes any) (any, bool) { +func (pc *instancePlanCache) getPlanFromList(headNode *instancePCNode, paramTypes any) (any, bool) { for node := headNode.next.Load(); node != nil; node = node.next.Load() { if checkTypesCompatibility4PC(node.value.paramTypes, paramTypes) { // v.Plan is read-only, no need to lock - node.lastUsed.Store(time.Now()) // atomically update the lastUsed field + if !pc.inEvict.Load() { + node.lastUsed.Store(time.Now()) // atomically update the lastUsed field + } return node.value, true } } @@ -97,6 +101,9 @@ func (*instancePlanCache) getPlanFromList(headNode *instancePCNode, paramTypes a // Put puts the key and values into the cache. // Due to some thread-safety issues, this Put operation might fail, use the returned succ to indicate it. func (pc *instancePlanCache) Put(key string, value, paramTypes any) (succ bool) { + if pc.inEvict.Load() { + return // do nothing if eviction is in progress + } vMem := value.(*PlanCacheValue).MemoryUsage() if vMem+pc.totCost.Load() > pc.hardMemLimit.Load() { return // do nothing if it exceeds the hard limit @@ -108,6 +115,9 @@ func (pc *instancePlanCache) Put(key string, value, paramTypes any) (succ bool) if _, ok := pc.getPlanFromList(headNode, paramTypes); ok { return // some other thread has inserted the same plan before } + if pc.inEvict.Load() { + return // do nothing if eviction is in progress + } firstNode := headNode.next.Load() currNode := pc.createNode(value) @@ -124,11 +134,15 @@ func (pc *instancePlanCache) Put(key string, value, paramTypes any) (succ bool) // step 1: iterate all values to collect their last_used // step 2: estimate an eviction threshold time based on all last_used values // step 3: iterate all values again and evict qualified values -func (pc *instancePlanCache) Evict() (evicted bool) { +func (pc *instancePlanCache) Evict() (detailInfo string, numEvicted int) { pc.evictMutex.Lock() // make sure only one thread to trigger eviction for safety defer pc.evictMutex.Unlock() - if pc.totCost.Load() < pc.softMemLimit.Load() { - return // do nothing + pc.inEvict.Store(true) + defer pc.inEvict.Store(false) + currentTot, softLimit := pc.totCost.Load(), pc.softMemLimit.Load() + if currentTot < softLimit { + detailInfo = fmt.Sprintf("memory usage is below the soft limit, currentTot: %v, softLimit: %v", currentTot, softLimit) + return } lastUsedTimes := make([]time.Time, 0, 64) pc.foreach(func(_, this *instancePCNode) bool { // step 1 @@ -136,12 +150,13 @@ func (pc *instancePlanCache) Evict() (evicted bool) { return false }) threshold := pc.calcEvictionThreshold(lastUsedTimes) // step 2 - pc.foreach(func(prev, this *instancePCNode) bool { // step 3 + detailInfo = fmt.Sprintf("evict threshold: %v", threshold) + pc.foreach(func(prev, this *instancePCNode) bool { // step 3 if !this.lastUsed.Load().After(threshold) { // if lastUsed<=threshold, evict this value if prev.next.CompareAndSwap(this, this.next.Load()) { // have to use CAS since pc.totCost.Sub(this.value.MemoryUsage()) // it might have been updated by other thread pc.totPlan.Sub(1) - evicted = true + numEvicted++ return true } } diff --git a/pkg/planner/core/plan_cache_instance_test.go b/pkg/planner/core/plan_cache_instance_test.go index ad6ff77ba7382..d9bbc7a846601 100644 --- a/pkg/planner/core/plan_cache_instance_test.go +++ b/pkg/planner/core/plan_cache_instance_test.go @@ -84,7 +84,8 @@ func TestInstancePlanCacheBasic(t *testing.T) { _hit(t, pc, 1, 0) // access 1-3 to refresh their last_used _hit(t, pc, 2, 0) _hit(t, pc, 3, 0) - require.Equal(t, pc.Evict(), true) + _, numEvicted := pc.Evict() + require.Equal(t, numEvicted > 0, true) require.Equal(t, pc.MemUsage(), int64(300)) _hit(t, pc, 1, 0) // access 1-3 to refresh their last_used _hit(t, pc, 2, 0) @@ -97,7 +98,8 @@ func TestInstancePlanCacheBasic(t *testing.T) { _put(pc, 1, 100, 0) _put(pc, 2, 100, 0) _put(pc, 3, 100, 0) - require.Equal(t, pc.Evict(), false) + _, numEvicted = pc.Evict() + require.Equal(t, numEvicted > 0, false) require.Equal(t, pc.MemUsage(), int64(300)) _hit(t, pc, 1, 0) _hit(t, pc, 2, 0) @@ -113,7 +115,8 @@ func TestInstancePlanCacheBasic(t *testing.T) { numHeads := 0 pcImpl.heads.Range(func(k, v any) bool { numHeads++; return true }) require.Equal(t, numHeads, 3) - require.Equal(t, pc.Evict(), true) + _, numEvicted = pc.Evict() + require.Equal(t, numEvicted > 0, true) require.Equal(t, pc.MemUsage(), int64(0)) numHeads = 0 pcImpl.heads.Range(func(k, v any) bool { numHeads++; return true }) @@ -174,7 +177,8 @@ func TestInstancePlanCacheWithMatchOpts(t *testing.T) { _hit(t, pc, 1, 1) // refresh 1-3's last_used _hit(t, pc, 1, 2) _hit(t, pc, 1, 3) - require.True(t, pc.Evict()) + _, numEvicted := pc.Evict() + require.True(t, numEvicted > 0) require.Equal(t, pc.MemUsage(), int64(300)) _hit(t, pc, 1, 1) _hit(t, pc, 1, 2) diff --git a/pkg/sessionctx/context.go b/pkg/sessionctx/context.go index 9702b04974e88..cb4810b303b5b 100644 --- a/pkg/sessionctx/context.go +++ b/pkg/sessionctx/context.go @@ -71,7 +71,7 @@ type InstancePlanCache interface { // Put puts the key and value into the cache. Put(key string, value, paramTypes any) (succ bool) // Evict evicts some cached values. - Evict() (evicted bool) + Evict() (detailInfo string, numEvicted int) // Size returns the number of cached values. Size() int64 // MemUsage returns the total memory usage of this plan cache.