Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

*: change default charset and collation from 'utf8 utf8_bin' to 'utf8mb4 utf8mb4_bin' #7965

Merged
merged 17 commits into from
Nov 9, 2018
Merged

*: change default charset and collation from 'utf8 utf8_bin' to 'utf8mb4 utf8mb4_bin' #7965

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
3435aae
*: change default charset and collation from 'utf8 utf8_bin' to 'utf8…
winkyao Oct 19, 2018
cd01346
address comment
winkyao Oct 22, 2018
6587484
Merge remote-tracking branch 'upstream/master' into charset_mb4
winkyao Oct 22, 2018
f4f840f
Merge remote-tracking branch 'upstream/master' into charset_mb4
winkyao Oct 24, 2018
cd57638
Merge remote-tracking branch 'upstream/master' into charset_mb4
winkyao Nov 5, 2018
d30424c
remove parser
winkyao Nov 5, 2018
a7e2183
remove util/charset
winkyao Nov 5, 2018
2ae3370
remove mysql
winkyao Nov 5, 2018
e54162e
fix ci
winkyao Nov 5, 2018
16c8cb0
Merge branch 'master' into charset_mb4
winkyao Nov 8, 2018
fdbc6e4
update go module
winkyao Nov 8, 2018
27ad600
Merge branch 'charset_mb4' of https://github.com/winkyao/tidb into ch…
winkyao Nov 8, 2018
68ec462
fix ci
winkyao Nov 8, 2018
c0995b7
address comments
winkyao Nov 9, 2018
5c1678d
address comment
winkyao Nov 9, 2018
90b3a60
cleanup
winkyao Nov 9, 2018
8efce03
Merge branch 'master' into charset_mb4
zz-jason Nov 9, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ddl/column_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -915,7 +915,7 @@ func (s *testColumnSuite) TestModifyColumn(c *C) {
{"int", "int unsigned", errUnsupportedModifyColumn.GenWithStackByArgs("length 10 is less than origin 11")},
{"varchar(10)", "text", nil},
{"varbinary(10)", "blob", nil},
{"text", "blob", errUnsupportedModifyColumn.GenWithStackByArgs("charset binary not match origin utf8")},
{"text", "blob", errUnsupportedModifyColumn.GenWithStackByArgs("charset binary not match origin utf8mb4")},
{"varchar(10)", "varchar(8)", errUnsupportedModifyColumn.GenWithStackByArgs("length 8 is less than origin 10")},
{"varchar(10)", "varchar(11)", nil},
{"varchar(10) character set utf8 collate utf8_bin", "varchar(10) character set utf8", nil},
Expand Down
6 changes: 3 additions & 3 deletions ddl/db_change_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,9 @@ func (s *testStateChangeSuite) TestShowCreateTable(c *C) {
got := result.Rows()[0][1]
var expected string
if job.Type == model.ActionAddIndex {
expected = "CREATE TABLE `t` (\n `id` int(11) DEFAULT NULL,\n KEY `idx` (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin"
expected = "CREATE TABLE `t` (\n `id` int(11) DEFAULT NULL,\n KEY `idx` (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"
} else if job.Type == model.ActionAddColumn {
expected = "CREATE TABLE `t` (\n `id` int(11) DEFAULT NULL,\n KEY `idx` (`id`),\n KEY `idx1` (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin"
expected = "CREATE TABLE `t` (\n `id` int(11) DEFAULT NULL,\n KEY `idx` (`id`),\n KEY `idx1` (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"
}
if got != expected {
checkErr = errors.Errorf("got %s, expected %s", got, expected)
Expand Down Expand Up @@ -376,7 +376,7 @@ func (s *testStateChangeSuite) TestAppendEnum(c *C) {
c.Assert(err.Error(), Equals, "[ddl:203]unsupported modify column the number of enum column's elements is less than the original: 2")
failAlterTableSQL2 := "alter table t change c2 c2 int default 0"
_, err = s.se.Execute(context.Background(), failAlterTableSQL2)
c.Assert(err.Error(), Equals, "[ddl:203]unsupported modify column charset binary not match origin utf8")
c.Assert(err.Error(), Equals, "[ddl:203]unsupported modify column charset binary not match origin utf8mb4")
alterTableSQL := "alter table t change c2 c2 enum('N','Y','A') DEFAULT 'A'"
_, err = s.se.Execute(context.Background(), alterTableSQL)
c.Assert(err, IsNil)
Expand Down
26 changes: 13 additions & 13 deletions ddl/db_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1321,7 +1321,7 @@ func (s *testDBSuite) TestAlterColumn(c *C) {
c.Assert(err, NotNil)
result := s.tk.MustQuery("show create table mc")
createSQL := result.Rows()[0][1]
expected := "CREATE TABLE `mc` (\n `a` int(11) NOT NULL,\n `b` int(11) DEFAULT NULL,\n `c` int(11) DEFAULT NULL,\n PRIMARY KEY (`a`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin"
expected := "CREATE TABLE `mc` (\n `a` int(11) NOT NULL,\n `b` int(11) DEFAULT NULL,\n `c` int(11) DEFAULT NULL,\n PRIMARY KEY (`a`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"
c.Assert(createSQL, Equals, expected)

// Change / modify column should preserve index options.
Expand All @@ -1332,7 +1332,7 @@ func (s *testDBSuite) TestAlterColumn(c *C) {
s.mustExec(c, "alter table mc modify column c bigint") // Unique should be preserved
result = s.tk.MustQuery("show create table mc")
createSQL = result.Rows()[0][1]
expected = "CREATE TABLE `mc` (\n `a` bigint(20) NOT NULL,\n `b` bigint(20) DEFAULT NULL,\n `c` bigint(20) DEFAULT NULL,\n PRIMARY KEY (`a`),\n UNIQUE KEY `c` (`c`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin"
expected = "CREATE TABLE `mc` (\n `a` bigint(20) NOT NULL,\n `b` bigint(20) DEFAULT NULL,\n `c` bigint(20) DEFAULT NULL,\n PRIMARY KEY (`a`),\n UNIQUE KEY `c` (`c`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"
c.Assert(createSQL, Equals, expected)

// Dropping or keeping auto_increment is allowed, however adding is not allowed.
Expand All @@ -1341,11 +1341,11 @@ func (s *testDBSuite) TestAlterColumn(c *C) {
s.mustExec(c, "alter table mc modify column a bigint auto_increment") // Keeps auto_increment
result = s.tk.MustQuery("show create table mc")
createSQL = result.Rows()[0][1]
expected = "CREATE TABLE `mc` (\n `a` bigint(20) NOT NULL AUTO_INCREMENT,\n `b` int(11) DEFAULT NULL,\n PRIMARY KEY (`a`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin"
expected = "CREATE TABLE `mc` (\n `a` bigint(20) NOT NULL AUTO_INCREMENT,\n `b` int(11) DEFAULT NULL,\n PRIMARY KEY (`a`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"
s.mustExec(c, "alter table mc modify column a bigint") // Drops auto_increment
result = s.tk.MustQuery("show create table mc")
createSQL = result.Rows()[0][1]
expected = "CREATE TABLE `mc` (\n `a` bigint(20) NOT NULL,\n `b` int(11) DEFAULT NULL,\n PRIMARY KEY (`a`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin"
expected = "CREATE TABLE `mc` (\n `a` bigint(20) NOT NULL,\n `b` int(11) DEFAULT NULL,\n PRIMARY KEY (`a`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"
c.Assert(createSQL, Equals, expected)
_, err = s.tk.Exec("alter table mc modify column a bigint auto_increment") // Adds auto_increment should throw error
c.Assert(err, NotNil)
Expand Down Expand Up @@ -1761,7 +1761,7 @@ func (s *testDBSuite) TestCreateTableWithPartition(c *C) {
partition p0 values less than (to_seconds('2004-01-01')),
partition p1 values less than (to_seconds('2005-01-01')));`)
s.tk.MustQuery("show create table t26").Check(
testkit.Rows("t26 CREATE TABLE `t26` (\n `a` date DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin\nPARTITION BY RANGE ( to_seconds(`a`) ) (\n PARTITION p0 VALUES LESS THAN (63240134400),\n PARTITION p1 VALUES LESS THAN (63271756800)\n)"))
testkit.Rows("t26 CREATE TABLE `t26` (\n `a` date DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\nPARTITION BY RANGE ( to_seconds(`a`) ) (\n PARTITION p0 VALUES LESS THAN (63240134400),\n PARTITION p1 VALUES LESS THAN (63271756800)\n)"))
s.tk.MustExec(`create table t27 (a bigint unsigned not null)
partition by range(a) (
partition p0 values less than (10),
Expand Down Expand Up @@ -2111,7 +2111,7 @@ func (s *testDBSuite) TestChangeColumnPosition(c *C) {
" `c` int(11) DEFAULT NULL,",
" `a` int(11) DEFAULT NULL,",
" KEY `t` (`c`)",
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin",
}
c.Assert(createSQL, Equals, strings.Join(exceptedSQL, "\n"))
}
Expand All @@ -2130,7 +2130,7 @@ func (s *testDBSuite) TestGeneratedColumnDDL(c *C) {
// Check show create table with virtual generated column.
result = s.tk.MustQuery(`show create table test_gv_ddl`)
result.Check(testkit.Rows(
"test_gv_ddl CREATE TABLE `test_gv_ddl` (\n `a` int(11) DEFAULT NULL,\n `b` int(11) GENERATED ALWAYS AS (`a` + 8) VIRTUAL DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin",
"test_gv_ddl CREATE TABLE `test_gv_ddl` (\n `a` int(11) DEFAULT NULL,\n `b` int(11) GENERATED ALWAYS AS (`a` + 8) VIRTUAL DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin",
))

// Check alter table add a stored generated column.
Expand All @@ -2144,7 +2144,7 @@ func (s *testDBSuite) TestGeneratedColumnDDL(c *C) {
result.Check(testkit.Rows("table_with_gen_col_blanks CREATE TABLE `table_with_gen_col_blanks` (\n" +
" `a` int(11) DEFAULT NULL,\n" +
" `b` char(20) GENERATED ALWAYS AS (CAST(`a` AS CHAR)) VIRTUAL DEFAULT NULL\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin"))
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))

genExprTests := []struct {
stmt string
Expand Down Expand Up @@ -2295,7 +2295,7 @@ func (s *testDBSuite) TestCheckColumnDefaultValue(c *C) {
s.tk.MustQuery(`show create table text_default_text`).Check(testutil.RowsWithSep("|",
"text_default_text CREATE TABLE `text_default_text` (\n"+
" `c1` text NOT NULL\n"+
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin",
))
ctx := s.tk.Se.(sessionctx.Context)
is := domain.GetDomain(ctx).InfoSchema()
Expand All @@ -2308,7 +2308,7 @@ func (s *testDBSuite) TestCheckColumnDefaultValue(c *C) {
s.tk.MustQuery(`show create table text_default_blob`).Check(testutil.RowsWithSep("|",
"text_default_blob CREATE TABLE `text_default_blob` (\n"+
" `c1` blob NOT NULL\n"+
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin",
))
is = domain.GetDomain(ctx).InfoSchema()
tblInfo, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("text_default_blob"))
Expand All @@ -2320,7 +2320,7 @@ func (s *testDBSuite) TestCheckColumnDefaultValue(c *C) {
s.tk.MustQuery(`show create table text_default_json`).Check(testutil.RowsWithSep("|",
"text_default_json CREATE TABLE `text_default_json` (\n"+
" `c1` json NOT NULL DEFAULT 'null'\n"+
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin",
))
is = domain.GetDomain(ctx).InfoSchema()
tblInfo, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("text_default_json"))
Expand All @@ -2335,8 +2335,8 @@ func (s *testDBSuite) TestCharacterSetInColumns(c *C) {
s.tk.MustExec("use varchar_test")
s.tk.MustExec("drop table if exists t")
s.tk.MustExec("create table t (c1 int, s1 varchar(10), s2 text)")
s.tk.MustQuery("select count(*) from information_schema.columns where table_schema = 'varchar_test' and character_set_name != 'utf8'").Check(testkit.Rows("0"))
s.tk.MustQuery("select count(*) from information_schema.columns where table_schema = 'varchar_test' and character_set_name = 'utf8'").Check(testkit.Rows("2"))
s.tk.MustQuery("select count(*) from information_schema.columns where table_schema = 'varchar_test' and character_set_name != 'utf8mb4'").Check(testkit.Rows("0"))
s.tk.MustQuery("select count(*) from information_schema.columns where table_schema = 'varchar_test' and character_set_name = 'utf8mb4'").Check(testkit.Rows("2"))

s.tk.MustExec("drop table if exists t5")
s.tk.MustExec("create table t5(id int) charset=UTF8;")
Expand Down
11 changes: 2 additions & 9 deletions ddl/ddl_api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func (d *ddl) CreateSchema(ctx sessionctx.Context, schema model.CIStr, charsetIn
dbInfo.Charset = charsetInfo.Chs
dbInfo.Collate = charsetInfo.Col
} else {
dbInfo.Charset, dbInfo.Collate = getDefaultCharsetAndCollate()
dbInfo.Charset, dbInfo.Collate = charset.GetDefaultCharsetAndCollate()
}

job := &model.Job{
Expand Down Expand Up @@ -118,13 +118,6 @@ func checkTooLongIndex(index model.CIStr) error {
return nil
}

func getDefaultCharsetAndCollate() (string, string) {
// TODO: TableDefaultCharset-->DatabaseDefaultCharset-->SystemDefaultCharset.
// TODO: Change TableOption parser to parse collate.
// This is a tmp solution.
return "utf8", "utf8_bin"
}

func setColumnFlagWithConstraint(colMap map[string]*table.Column, v *ast.Constraint) {
switch v.Tp {
case ast.ConstraintPrimaryKey:
Expand Down Expand Up @@ -204,7 +197,7 @@ func setCharsetCollationFlenDecimal(tp *types.FieldType) error {
if len(tp.Charset) == 0 {
switch tp.Tp {
case mysql.TypeString, mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeEnum, mysql.TypeSet:
tp.Charset, tp.Collate = getDefaultCharsetAndCollate()
tp.Charset, tp.Collate = charset.GetDefaultCharsetAndCollate()
default:
tp.Charset = charset.CharsetBin
tp.Collate = charset.CharsetBin
Expand Down
2 changes: 1 addition & 1 deletion executor/ddl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ func (s *testSuite) TestAlterTableModifyColumn(c *C) {
tk.MustExec("alter table mc modify column c2 text")
result := tk.MustQuery("show create table mc")
createSQL := result.Rows()[0][1]
expected := "CREATE TABLE `mc` (\n `c1` bigint(20) DEFAULT NULL,\n `c2` text DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin"
expected := "CREATE TABLE `mc` (\n `c1` bigint(20) DEFAULT NULL,\n `c2` text DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"
c.Assert(createSQL, Equals, expected)
}

Expand Down
6 changes: 3 additions & 3 deletions executor/executor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1005,7 +1005,7 @@ func (s *testSuite) TestUnion(c *C) {
tk.MustExec("CREATE TABLE t1 (a date)")
tk.MustExec("CREATE TABLE t2 (a date)")
tk.MustExec("SELECT a from t1 UNION select a FROM t2")
tk.MustQuery("show create table t1").Check(testkit.Rows("t1 CREATE TABLE `t1` (\n" + " `a` date DEFAULT NULL\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin"))
tk.MustQuery("show create table t1").Check(testkit.Rows("t1 CREATE TABLE `t1` (\n" + " `a` date DEFAULT NULL\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))

// Move from session test.
tk.MustExec("drop table if exists t1, t2")
Expand Down Expand Up @@ -1767,8 +1767,8 @@ func (s *testSuite) TestTableScan(c *C) {
c.Assert(len(result.Rows()), GreaterEqual, 4)
tk.MustExec("use test")
tk.MustExec("create database mytest")
rowStr1 := fmt.Sprintf("%s %s %s %s %v", "def", "mysql", "utf8", "utf8_bin", nil)
rowStr2 := fmt.Sprintf("%s %s %s %s %v", "def", "mytest", "utf8", "utf8_bin", nil)
rowStr1 := fmt.Sprintf("%s %s %s %s %v", "def", "mysql", "utf8mb4", "utf8mb4_bin", nil)
rowStr2 := fmt.Sprintf("%s %s %s %s %v", "def", "mytest", "utf8mb4", "utf8mb4_bin", nil)
tk.MustExec("use information_schema")
result = tk.MustQuery("select * from schemata where schema_name = 'mysql'")
result.Check(testkit.Rows(rowStr1))
Expand Down
2 changes: 1 addition & 1 deletion executor/show.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -618,7 +618,7 @@ func (e *ShowExec) fetchShowCreateTable() error {
buf.WriteString(") ENGINE=InnoDB")
charsetName := tb.Meta().Charset
if len(charsetName) == 0 {
charsetName = charset.CharsetUTF8
charsetName = charset.CharsetUTF8MB4
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

s/charset.CharsetUTF8MB4/mysql.DefaultCharset/

}
collate := tb.Meta().Collate
// Set default collate if collate is not specified.
Expand Down
Loading