diff --git a/executor/analyze.go b/executor/analyze.go
index b0568abefd789..2e55fde99b43b 100644
--- a/executor/analyze.go
+++ b/executor/analyze.go
@@ -1183,7 +1183,7 @@ type analyzeIndexIncrementalExec struct {
 
 func analyzeIndexIncremental(idxExec *analyzeIndexIncrementalExec) analyzeResult {
 	startPos := idxExec.oldHist.GetUpper(idxExec.oldHist.Len() - 1)
-	values, err := codec.DecodeRange(startPos.GetBytes(), len(idxExec.idxInfo.Columns))
+	values, _, err := codec.DecodeRange(startPos.GetBytes(), len(idxExec.idxInfo.Columns))
 	if err != nil {
 		return analyzeResult{Err: err, job: idxExec.job}
 	}
diff --git a/statistics/feedback.go b/statistics/feedback.go
index 99a1596bd4e1b..992bb3922325a 100644
--- a/statistics/feedback.go
+++ b/statistics/feedback.go
@@ -121,11 +121,11 @@ func (q *QueryFeedback) DecodeToRanges(isIndex bool) ([]*ranger.Range, error) {
 		if isIndex {
 			var err error
 			// As we do not know the origin length, just use a custom value here.
-			lowVal, err = codec.DecodeRange(low.GetBytes(), 4)
+			lowVal, _, err = codec.DecodeRange(low.GetBytes(), 4)
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
-			highVal, err = codec.DecodeRange(high.GetBytes(), 4)
+			highVal, _, err = codec.DecodeRange(high.GetBytes(), 4)
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
@@ -837,7 +837,7 @@ func ConvertDatumsType(vals []types.Datum, ft *types.FieldType, loc *time.Locati
 }
 
 func decodeColumnBounds(data []byte, ft *types.FieldType) ([]types.Datum, error) {
-	vals, err := codec.DecodeRange(data, 1)
+	vals, _, err := codec.DecodeRange(data, 1)
 	if err != nil {
 		return nil, err
 	}
diff --git a/statistics/histogram.go b/statistics/histogram.go
index 73adeee757bac..659793dfaa3b4 100644
--- a/statistics/histogram.go
+++ b/statistics/histogram.go
@@ -216,9 +216,10 @@ func ValueToString(value *types.Datum, idxCols int) (string, error) {
 	if idxCols == 0 {
 		return value.ToString()
 	}
-	decodedVals, err := codec.DecodeRange(value.GetBytes(), idxCols)
-	if err != nil {
-		return "", errors.Trace(err)
+	// Treat remaining part that cannot decode successfully as bytes.
+	decodedVals, remained, err := codec.DecodeRange(value.GetBytes(), idxCols)
+	if err != nil && len(remained) > 0 {
+		decodedVals = append(decodedVals, types.NewBytesDatum(remained))
 	}
 	str, err := types.DatumsToString(decodedVals, true)
 	if err != nil {
diff --git a/statistics/histogram_test.go b/statistics/histogram_test.go
index e131b143e7306..68b8c55f2ce33 100644
--- a/statistics/histogram_test.go
+++ b/statistics/histogram_test.go
@@ -120,3 +120,14 @@ num: 30 lower_bound: 12 upper_bound: 14 repeats: 10`
 	c.Assert(err, IsNil, Commentf("Test failed: %v", err))
 	c.Assert(newIdx.String(), Equals, idxResult)
 }
+
+func (s *testStatisticsSuite) TestValueToString4InvalidKey(c *C) {
+	bytes, err := codec.EncodeKey(nil, nil, types.NewDatum(1), types.NewDatum(0.5))
+	c.Assert(err, IsNil)
+	// Append invalid flag.
+	bytes = append(bytes, 20)
+	datum := types.NewDatum(bytes)
+	res, err := ValueToString(&datum, 3)
+	c.Assert(err, IsNil)
+	c.Assert(res, Equals, "(1, 0.5, \x14)")
+}
diff --git a/util/codec/codec.go b/util/codec/codec.go
index 19cbfff6c7977..f2a9b323b3f5b 100644
--- a/util/codec/codec.go
+++ b/util/codec/codec.go
@@ -312,9 +312,9 @@ func Decode(b []byte, size int) ([]types.Datum, error) {
 
 // DecodeRange decodes the range values from a byte slice that generated by EncodeKey.
 // It handles some special values like `MinNotNull` and `MaxValueDatum`.
-func DecodeRange(b []byte, size int) ([]types.Datum, error) {
+func DecodeRange(b []byte, size int) ([]types.Datum, []byte, error) {
 	if len(b) < 1 {
-		return nil, errors.New("invalid encoded key: length of key is zero")
+		return nil, b, errors.New("invalid encoded key: length of key is zero")
 	}
 
 	var (
@@ -326,7 +326,7 @@ func DecodeRange(b []byte, size int) ([]types.Datum, error) {
 		var d types.Datum
 		b, d, err = DecodeOne(b)
 		if err != nil {
-			return nil, errors.Trace(err)
+			return values, b, errors.Trace(err)
 		}
 		values = append(values, d)
 	}
@@ -341,10 +341,10 @@ func DecodeRange(b []byte, size int) ([]types.Datum, error) {
 		case maxFlag, maxFlag + 1:
 			values = append(values, types.MaxValueDatum())
 		default:
-			return nil, errors.Errorf("invalid encoded key flag %v", b[0])
+			return values, b, errors.Errorf("invalid encoded key flag %v", b[0])
 		}
 	}
-	return values, nil
+	return values, nil, nil
 }
 
 // DecodeOne decodes on datum from a byte slice generated with EncodeKey or EncodeValue.
diff --git a/util/codec/codec_test.go b/util/codec/codec_test.go
index b54a6ae8adeb4..90c84869ad18e 100644
--- a/util/codec/codec_test.go
+++ b/util/codec/codec_test.go
@@ -985,14 +985,14 @@ func chunkForTest(c *C, sc *stmtctx.StatementContext, datums []types.Datum, tps
 }
 
 func (s *testCodecSuite) TestDecodeRange(c *C) {
-	_, err := DecodeRange(nil, 0)
+	_, _, err := DecodeRange(nil, 0)
 	c.Assert(err, NotNil)
 
 	datums := types.MakeDatums(1, "abc", 1.1, []byte("def"))
 	rowData, err := EncodeValue(nil, nil, datums...)
 	c.Assert(err, IsNil)
 
-	datums1, err := DecodeRange(rowData, len(datums))
+	datums1, _, err := DecodeRange(rowData, len(datums))
 	c.Assert(err, IsNil)
 	for i, datum := range datums1 {
 		cmp, err := datum.CompareDatum(nil, &datums[i])
@@ -1002,7 +1002,7 @@ func (s *testCodecSuite) TestDecodeRange(c *C) {
 
 	for _, b := range []byte{NilFlag, bytesFlag, maxFlag, maxFlag + 1} {
 		newData := append(rowData, b)
-		_, err := DecodeRange(newData, len(datums)+1)
+		_, _, err := DecodeRange(newData, len(datums)+1)
 		c.Assert(err, IsNil)
 	}
 }