From c8f2b0a6a2c96f25b4d6e03ea8d083e63854fd46 Mon Sep 17 00:00:00 2001 From: Jianjun Liao Date: Tue, 4 Jun 2024 15:02:08 +0800 Subject: [PATCH 1/8] set memory limit Signed-off-by: Jianjun Liao --- br/cmd/br/cmd.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/br/cmd/br/cmd.go b/br/cmd/br/cmd.go index df0395fa1d719..712455dde6e56 100644 --- a/br/cmd/br/cmd.go +++ b/br/cmd/br/cmd.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "runtime/debug" "sync" "sync/atomic" "time" @@ -24,6 +25,7 @@ import ( "github.com/pingcap/tidb/pkg/util/memory" "github.com/pingcap/tidb/pkg/util/redact" "github.com/spf13/cobra" + "go.uber.org/zap" ) var ( @@ -162,6 +164,31 @@ func Init(cmd *cobra.Command) (err error) { } log.ReplaceGlobals(lg, p) memory.InitMemoryHook() + if debug.SetMemoryLimit(-1) == 0 { + memtotal, e := memory.MemTotal() + if e != nil { + err = e + return + } + memused, e := memory.MemUsed() + if e != nil { + err = e + return + } + halfGB := uint64(512 * 1024 * 1024) + // 0 memused memtotal-512MB memtotal + // +--------+--------------------+--------------+ + // br mem upper limit + memleft := memtotal - memused - halfGB + // 0 512MB +inf + // +--------+--------------------> + // ^____________________> + // GOMEMLIMIT range + if memleft > halfGB { + log.Info("set memory limit", zap.Uint64("limit", memleft)) + debug.SetMemoryLimit(int64(memleft)) + } + } redactLog, e := cmd.Flags().GetBool(FlagRedactLog) if e != nil { From 5ac7699c1df6db7bc1cd2c7784985a99cc59dfe6 Mon Sep 17 00:00:00 2001 From: Jianjun Liao Date: Tue, 4 Jun 2024 16:13:39 +0800 Subject: [PATCH 2/8] set memory limit Signed-off-by: Jianjun Liao --- br/cmd/br/cmd.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/br/cmd/br/cmd.go b/br/cmd/br/cmd.go index 712455dde6e56..43be8cab3b74f 100644 --- a/br/cmd/br/cmd.go +++ b/br/cmd/br/cmd.go @@ -5,6 +5,7 @@ package main import ( "context" "fmt" + "math" "os" "path/filepath" "runtime/debug" @@ -164,7 +165,7 @@ func Init(cmd *cobra.Command) (err error) { } log.ReplaceGlobals(lg, p) memory.InitMemoryHook() - if debug.SetMemoryLimit(-1) == 0 { + if debug.SetMemoryLimit(-1) == math.MaxInt64 { memtotal, e := memory.MemTotal() if e != nil { err = e @@ -184,6 +185,7 @@ func Init(cmd *cobra.Command) (err error) { // +--------+--------------------> // ^____________________> // GOMEMLIMIT range + log.Info("calculate the rest memory", zap.Uint64("memtotal", memtotal), zap.Uint64("memused", memused)) if memleft > halfGB { log.Info("set memory limit", zap.Uint64("limit", memleft)) debug.SetMemoryLimit(int64(memleft)) From 55bc0b4465ff56398afc5b47b7835ac43afe4781 Mon Sep 17 00:00:00 2001 From: Jianjun Liao Date: Wed, 5 Jun 2024 15:16:49 +0800 Subject: [PATCH 3/8] calculate the memory limit Signed-off-by: Jianjun Liao --- br/cmd/br/cmd.go | 36 ++++++++++++++++++++++-------------- br/cmd/br/main_test.go | 16 ++++++++++++++++ 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/br/cmd/br/cmd.go b/br/cmd/br/cmd.go index 43be8cab3b74f..e3ea207bcfa99 100644 --- a/br/cmd/br/cmd.go +++ b/br/cmd/br/cmd.go @@ -110,6 +110,23 @@ func AddFlags(cmd *cobra.Command) { _ = cmd.PersistentFlags().MarkHidden(FlagRedactLog) } +func calculateMemoryLimit(memleft uint64) uint64 { + const halfGB = uint64(512 * 1024 * 1024) + const fourGB = 8 * halfGB + // memreserved = f(memleft) = 512MB * memleft / (memleft + 4GB) + // * f(0) = 0 + // * f(4GB) = 256MB + // * f(+inf) -> 512MB + memreserved := halfGB / (1 + fourGB/(memleft+1)) + // 0 memused memtotal-memreserved memtotal + // +--------+--------------------+----------------+ + // ^ br mem upper limit + // +--------------------^ + // GOMEMLIMIT range + memlimit := memleft - memreserved + return memlimit +} + // Init initializes BR cli. func Init(cmd *cobra.Command) (err error) { initOnce.Do(func() { @@ -176,20 +193,11 @@ func Init(cmd *cobra.Command) (err error) { err = e return } - halfGB := uint64(512 * 1024 * 1024) - // 0 memused memtotal-512MB memtotal - // +--------+--------------------+--------------+ - // br mem upper limit - memleft := memtotal - memused - halfGB - // 0 512MB +inf - // +--------+--------------------> - // ^____________________> - // GOMEMLIMIT range - log.Info("calculate the rest memory", zap.Uint64("memtotal", memtotal), zap.Uint64("memused", memused)) - if memleft > halfGB { - log.Info("set memory limit", zap.Uint64("limit", memleft)) - debug.SetMemoryLimit(int64(memleft)) - } + memleft := memtotal - memused + memlimit := calculateMemoryLimit(memleft) + log.Info("calculate the rest memory", + zap.Uint64("memtotal", memtotal), zap.Uint64("memused", memused), zap.Uint64("memlimit", memlimit)) + debug.SetMemoryLimit(int64(memlimit)) } redactLog, e := cmd.Flags().GetBool(FlagRedactLog) diff --git a/br/cmd/br/main_test.go b/br/cmd/br/main_test.go index 5fa5b1439a2ec..bdd6f7282ee62 100644 --- a/br/cmd/br/main_test.go +++ b/br/cmd/br/main_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + "github.com/stretchr/testify/require" "go.uber.org/goleak" ) @@ -75,3 +76,18 @@ func TestRunMain(*testing.T) { <-waitCh } + +func TestCalculateMemoryLimit(t *testing.T) { + // f(0 Byte) = 0 Byte + require.Equal(t, uint64(0), calculateMemoryLimit(0)) + // f(100 KB) = 87.5 KB + require.Equal(t, uint64(89600), calculateMemoryLimit(100*1024)) + // f(100 MB) = 87.5 MB + require.Equal(t, uint64(91763188), calculateMemoryLimit(100*1024*1024)) + // f(3.99 GB) = 3.74 GB + require.Equal(t, uint64(4026531839), calculateMemoryLimit(4*1024*1024*1024-1)) + // f(4 GB) = 3.5 GB + require.Equal(t, uint64(3758096384), calculateMemoryLimit(4*1024*1024*1024)) + // f(32 GB) = 31.5 GB + require.Equal(t, uint64(33822867456), calculateMemoryLimit(32*1024*1024*1024)) +} From e01fe8bdcb451f4805b74e5c6f88ae606b70d5c7 Mon Sep 17 00:00:00 2001 From: Jianjun Liao Date: Wed, 5 Jun 2024 15:19:17 +0800 Subject: [PATCH 4/8] bazel prepare Signed-off-by: Jianjun Liao --- br/cmd/br/BUILD.bazel | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/br/cmd/br/BUILD.bazel b/br/cmd/br/BUILD.bazel index c6b4e1d646e11..ec61730baad0e 100644 --- a/br/cmd/br/BUILD.bazel +++ b/br/cmd/br/BUILD.bazel @@ -63,5 +63,8 @@ go_test( srcs = ["main_test.go"], embed = [":br_lib"], flaky = True, - deps = ["@org_uber_go_goleak//:goleak"], + deps = [ + "@com_github_stretchr_testify//require", + "@org_uber_go_goleak//:goleak", + ], ) From 438d462e5628ce40476d26e88e3f83ae63ce92ae Mon Sep 17 00:00:00 2001 From: Jianjun Liao Date: Fri, 19 Jul 2024 15:49:27 +0800 Subject: [PATCH 5/8] commit some suggestions Signed-off-by: Jianjun Liao --- br/cmd/br/cmd.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/br/cmd/br/cmd.go b/br/cmd/br/cmd.go index e3ea207bcfa99..9f76b77abc16e 100644 --- a/br/cmd/br/cmd.go +++ b/br/cmd/br/cmd.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/memory" "github.com/pingcap/tidb/pkg/util/redact" + "github.com/pingcap/tidb/pkg/util/size" "github.com/spf13/cobra" "go.uber.org/zap" ) @@ -110,14 +111,15 @@ func AddFlags(cmd *cobra.Command) { _ = cmd.PersistentFlags().MarkHidden(FlagRedactLog) } +const halfGiB uint64 = 512 * size.MB +const fourGiB uint64 = 8 * halfGiB + func calculateMemoryLimit(memleft uint64) uint64 { - const halfGB = uint64(512 * 1024 * 1024) - const fourGB = 8 * halfGB // memreserved = f(memleft) = 512MB * memleft / (memleft + 4GB) // * f(0) = 0 // * f(4GB) = 256MB // * f(+inf) -> 512MB - memreserved := halfGB / (1 + fourGB/(memleft+1)) + memreserved := halfGiB / (1 + fourGiB/(memleft|1)) // 0 memused memtotal-memreserved memtotal // +--------+--------------------+----------------+ // ^ br mem upper limit @@ -193,11 +195,15 @@ func Init(cmd *cobra.Command) (err error) { err = e return } - memleft := memtotal - memused - memlimit := calculateMemoryLimit(memleft) - log.Info("calculate the rest memory", - zap.Uint64("memtotal", memtotal), zap.Uint64("memused", memused), zap.Uint64("memlimit", memlimit)) - debug.SetMemoryLimit(int64(memlimit)) + if memused >= memtotal { + log.Warn("failed to obtain memory size, skip setting memory limit", zap.Uint64("memused", memused), zap.Uint64("memtotal", memtotal)) + } else { + memleft := memtotal - memused + memlimit := calculateMemoryLimit(memleft) + log.Info("calculate the rest memory and set memory limit", + zap.Uint64("memtotal", memtotal), zap.Uint64("memused", memused), zap.Uint64("memlimit", memlimit)) + debug.SetMemoryLimit(int64(memlimit)) + } } redactLog, e := cmd.Flags().GetBool(FlagRedactLog) From e1650d52b9bb6868b586882558fc67dee7f59cd5 Mon Sep 17 00:00:00 2001 From: Jianjun Liao Date: Fri, 19 Jul 2024 17:39:10 +0800 Subject: [PATCH 6/8] add the minimum value of memlimit Signed-off-by: Jianjun Liao --- br/cmd/br/BUILD.bazel | 2 ++ br/cmd/br/cmd.go | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/br/cmd/br/BUILD.bazel b/br/cmd/br/BUILD.bazel index ec61730baad0e..b82ecbd2bc8dc 100644 --- a/br/cmd/br/BUILD.bazel +++ b/br/cmd/br/BUILD.bazel @@ -37,9 +37,11 @@ go_library( "//pkg/util", "//pkg/util/gctuner", "//pkg/util/logutil", + "//pkg/util/mathutil", "//pkg/util/memory", "//pkg/util/metricsutil", "//pkg/util/redact", + "//pkg/util/size", "@com_github_gogo_protobuf//proto", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_kvproto//pkg/brpb", diff --git a/br/cmd/br/cmd.go b/br/cmd/br/cmd.go index 9f76b77abc16e..6d6d1780768fc 100644 --- a/br/cmd/br/cmd.go +++ b/br/cmd/br/cmd.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/tidb/pkg/config" tidbutils "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/logutil" + "github.com/pingcap/tidb/pkg/util/mathutil" "github.com/pingcap/tidb/pkg/util/memory" "github.com/pingcap/tidb/pkg/util/redact" "github.com/pingcap/tidb/pkg/util/size" @@ -111,8 +112,9 @@ func AddFlags(cmd *cobra.Command) { _ = cmd.PersistentFlags().MarkHidden(FlagRedactLog) } +const quarterGiB uint64 = 256 * size.MB const halfGiB uint64 = 512 * size.MB -const fourGiB uint64 = 8 * halfGiB +const fourGiB uint64 = 4 * size.GB func calculateMemoryLimit(memleft uint64) uint64 { // memreserved = f(memleft) = 512MB * memleft / (memleft + 4GB) @@ -200,6 +202,9 @@ func Init(cmd *cobra.Command) (err error) { } else { memleft := memtotal - memused memlimit := calculateMemoryLimit(memleft) + // BR command needs 256 MiB at least, if the left memory is less than 256 MiB, + // the memory limit cannot limit anyway and then finally OOM. + memlimit = mathutil.Max(memlimit, quarterGiB) log.Info("calculate the rest memory and set memory limit", zap.Uint64("memtotal", memtotal), zap.Uint64("memused", memused), zap.Uint64("memlimit", memlimit)) debug.SetMemoryLimit(int64(memlimit)) From d9c42cbc240c8a10adfbe6e8159d6e32348ee48f Mon Sep 17 00:00:00 2001 From: Jianjun Liao Date: Fri, 19 Jul 2024 18:04:00 +0800 Subject: [PATCH 7/8] commit some suggestions Signed-off-by: Jianjun Liao --- br/cmd/br/cmd.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/br/cmd/br/cmd.go b/br/cmd/br/cmd.go index 6d6d1780768fc..17e8af0fa272e 100644 --- a/br/cmd/br/cmd.go +++ b/br/cmd/br/cmd.go @@ -198,7 +198,8 @@ func Init(cmd *cobra.Command) (err error) { return } if memused >= memtotal { - log.Warn("failed to obtain memory size, skip setting memory limit", zap.Uint64("memused", memused), zap.Uint64("memtotal", memtotal)) + log.Warn("failed to obtain memory size, skip setting memory limit", + zap.Uint64("memused", memused), zap.Uint64("memtotal", memtotal)) } else { memleft := memtotal - memused memlimit := calculateMemoryLimit(memleft) @@ -207,7 +208,7 @@ func Init(cmd *cobra.Command) (err error) { memlimit = mathutil.Max(memlimit, quarterGiB) log.Info("calculate the rest memory and set memory limit", zap.Uint64("memtotal", memtotal), zap.Uint64("memused", memused), zap.Uint64("memlimit", memlimit)) - debug.SetMemoryLimit(int64(memlimit)) + debug.SetMemoryLimit(int64(memlimit & math.MaxInt64)) } } From 1e15feaa090b0bd3892ab3179735de9aa3ea5f3c Mon Sep 17 00:00:00 2001 From: Jianjun Liao Date: Fri, 19 Jul 2024 18:08:13 +0800 Subject: [PATCH 8/8] commit some suggestions Signed-off-by: Jianjun Liao --- br/cmd/br/cmd.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/br/cmd/br/cmd.go b/br/cmd/br/cmd.go index 17e8af0fa272e..02939c78b4676 100644 --- a/br/cmd/br/cmd.go +++ b/br/cmd/br/cmd.go @@ -206,9 +206,12 @@ func Init(cmd *cobra.Command) (err error) { // BR command needs 256 MiB at least, if the left memory is less than 256 MiB, // the memory limit cannot limit anyway and then finally OOM. memlimit = mathutil.Max(memlimit, quarterGiB) - log.Info("calculate the rest memory and set memory limit", + log.Info("calculate the rest memory", zap.Uint64("memtotal", memtotal), zap.Uint64("memused", memused), zap.Uint64("memlimit", memlimit)) - debug.SetMemoryLimit(int64(memlimit & math.MaxInt64)) + // No need to set memory limit because the left memory is sufficient. + if memlimit < uint64(math.MaxInt64) { + debug.SetMemoryLimit(int64(memlimit)) + } } }