Fix various quoted keys bugs (#400)

Fixes #396 #397 #398 #399

Author: x-hgg-x

keysparsing.go

@@ -5,7 +5,6 @@ package toml
 import (
 	"errors"
 	"fmt"
-	"unicode"
 )
 
 // Convert the bare key group string to an array.
@@ -109,5 +108,5 @@ func parseKey(key string) ([]string, error) {
 }
 
 func isValidBareChar(r rune) bool {
-	return isAlphanumeric(r) || r == '-' || unicode.IsNumber(r)
+	return isAlphanumeric(r) || r == '-' || isDigit(r)
 }

lexer_test.go

@@ -105,9 +105,9 @@ func TestBasicKeyWithUppercaseMix(t *testing.T) {
 }
 
 func TestBasicKeyWithInternationalCharacters(t *testing.T) {
-	testFlow(t, "héllÖ", []token{
-		{Position{1, 1}, tokenKey, "héllÖ"},
-		{Position{1, 6}, tokenEOF, ""},
+	testFlow(t, "'héllÖ'", []token{
+		{Position{1, 1}, tokenKey, "'héllÖ'"},
+		{Position{1, 8}, tokenEOF, ""},
 	})
 }
 

marshal.go

@@ -436,7 +436,7 @@ func (e *Encoder) valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, er
 					if tree, ok := val.(*Tree); ok && mtypef.Anonymous && !opts.nameFromTag && !e.promoteAnon {
 						e.appendTree(tval, tree)
 					} else {
-						tval.SetWithOptions(opts.name, SetOptions{
+						tval.SetPathWithOptions([]string{opts.name}, SetOptions{
 							Comment:   opts.comment,
 							Commented: opts.commented,
 							Multiline: opts.multiline,
@@ -481,7 +481,7 @@ func (e *Encoder) valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, er
 				}
 				tval.SetPath([]string{keyStr}, val)
 			} else {
-				tval.Set(key.String(), val)
+				tval.SetPath([]string{key.String()}, val)
 			}
 		}
 	}
@@ -754,17 +754,17 @@ func (d *Decoder) valueFromTree(mtype reflect.Type, tval *Tree, mval1 *reflect.V
 				found := false
 				if tval != nil {
 					for _, key := range keysToTry {
-						exists := tval.Has(key)
+						exists := tval.HasPath([]string{key})
 						if !exists {
 							continue
 						}
 
 						d.visitor.push(key)
-						val := tval.Get(key)
+						val := tval.GetPath([]string{key})
 						fval := mval.Field(i)
 						mvalf, err := d.valueFromToml(mtypef.Type, val, &fval)
 						if err != nil {
-							return mval, formatError(err, tval.GetPosition(key))
+							return mval, formatError(err, tval.GetPositionPath([]string{key}))
 						}
 						mval.Field(i).Set(mvalf)
 						found = true
@@ -838,7 +838,7 @@ func (d *Decoder) valueFromTree(mtype reflect.Type, tval *Tree, mval1 *reflect.V
 			val := tval.GetPath([]string{key})
 			mvalf, err := d.valueFromToml(mtype.Elem(), val, nil)
 			if err != nil {
-				return mval, formatError(err, tval.GetPosition(key))
+				return mval, formatError(err, tval.GetPositionPath([]string{key}))
 			}
 			mval.SetMapIndex(reflect.ValueOf(key).Convert(mtype.Key()), mvalf)
 			d.visitor.pop()

marshal_test.go

@@ -287,6 +287,59 @@ func TestBasicUnmarshal(t *testing.T) {
 	}
 }
 
+type quotedKeyMarshalTestStruct struct {
+	String  string                      `toml:"Z.string-àéù"`
+	Float   float64                     `toml:"Yfloat-𝟘"`
+	Sub     basicMarshalTestSubStruct   `toml:"Xsubdoc-àéù"`
+	SubList []basicMarshalTestSubStruct `toml:"W.sublist-𝟘"`
+}
+
+var quotedKeyMarshalTestData = quotedKeyMarshalTestStruct{
+	String:  "Hello",
+	Float:   3.5,
+	Sub:     basicMarshalTestSubStruct{"One"},
+	SubList: []basicMarshalTestSubStruct{{"Two"}, {"Three"}},
+}
+
+var quotedKeyMarshalTestToml = []byte(`"Yfloat-𝟘" = 3.5
+"Z.string-àéù" = "Hello"
+
+[["W.sublist-𝟘"]]
+  String2 = "Two"
+
+[["W.sublist-𝟘"]]
+  String2 = "Three"
+
+["Xsubdoc-àéù"]
+  String2 = "One"
+`)
+
+func TestBasicMarshalQuotedKey(t *testing.T) {
+	result, err := Marshal(quotedKeyMarshalTestData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := quotedKeyMarshalTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestBasicUnmarshalQuotedKey(t *testing.T) {
+	tree, err := LoadBytes(quotedKeyMarshalTestToml)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var q quotedKeyMarshalTestStruct
+	tree.Unmarshal(&q)
+	fmt.Println(q)
+
+	if !reflect.DeepEqual(quotedKeyMarshalTestData, q) {
+		t.Errorf("Bad unmarshal: expected\n-----\n%v\n-----\ngot\n-----\n%v\n-----\n", quotedKeyMarshalTestData, q)
+	}
+}
+
 type testDoc struct {
 	Title       string            `toml:"title"`
 	BasicLists  testDocBasicLists `toml:"basic_lists"`
@@ -2070,7 +2123,6 @@ func TestUnmarshalCamelCaseKey(t *testing.T) {
 	}
 }
 
-
 func TestUnmarshalNegativeUint(t *testing.T) {
 	type check struct{ U uint }
 

query/match.go

@@ -2,6 +2,7 @@ package query
 
 import (
 	"fmt"
+
 	"github.com/pelletier/go-toml"
 )
 
@@ -44,16 +45,16 @@ func newMatchKeyFn(name string) *matchKeyFn {
 func (f *matchKeyFn) call(node interface{}, ctx *queryContext) {
 	if array, ok := node.([]*toml.Tree); ok {
 		for _, tree := range array {
-			item := tree.Get(f.Name)
+			item := tree.GetPath([]string{f.Name})
 			if item != nil {
-				ctx.lastPosition = tree.GetPosition(f.Name)
+				ctx.lastPosition = tree.GetPositionPath([]string{f.Name})
 				f.next.call(item, ctx)
 			}
 		}
 	} else if tree, ok := node.(*toml.Tree); ok {
-		item := tree.Get(f.Name)
+		item := tree.GetPath([]string{f.Name})
 		if item != nil {
-			ctx.lastPosition = tree.GetPosition(f.Name)
+			ctx.lastPosition = tree.GetPositionPath([]string{f.Name})
 			f.next.call(item, ctx)
 		}
 	}
@@ -129,8 +130,8 @@ func newMatchAnyFn() *matchAnyFn {
 func (f *matchAnyFn) call(node interface{}, ctx *queryContext) {
 	if tree, ok := node.(*toml.Tree); ok {
 		for _, k := range tree.Keys() {
-			v := tree.Get(k)
-			ctx.lastPosition = tree.GetPosition(k)
+			v := tree.GetPath([]string{k})
+			ctx.lastPosition = tree.GetPositionPath([]string{k})
 			f.next.call(v, ctx)
 		}
 	}
@@ -168,8 +169,8 @@ func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) {
 		var visit func(tree *toml.Tree)
 		visit = func(tree *toml.Tree) {
 			for _, k := range tree.Keys() {
-				v := tree.Get(k)
-				ctx.lastPosition = tree.GetPosition(k)
+				v := tree.GetPath([]string{k})
+				ctx.lastPosition = tree.GetPositionPath([]string{k})
 				f.next.call(v, ctx)
 				switch node := v.(type) {
 				case *toml.Tree:
@@ -207,9 +208,9 @@ func (f *matchFilterFn) call(node interface{}, ctx *queryContext) {
 	switch castNode := node.(type) {
 	case *toml.Tree:
 		for _, k := range castNode.Keys() {
-			v := castNode.Get(k)
+			v := castNode.GetPath([]string{k})
 			if fn(v) {
-				ctx.lastPosition = castNode.GetPosition(k)
+				ctx.lastPosition = castNode.GetPositionPath([]string{k})
 				f.next.call(v, ctx)
 			}
 		}

query/tokens.go

@@ -2,9 +2,9 @@ package query
 
 import (
 	"fmt"
-	"github.com/pelletier/go-toml"
 	"strconv"
-	"unicode"
+
+	"github.com/pelletier/go-toml"
 )
 
 // Define tokens
@@ -92,11 +92,11 @@ func isSpace(r rune) bool {
 }
 
 func isAlphanumeric(r rune) bool {
-	return unicode.IsLetter(r) || r == '_'
+	return 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || r == '_'
 }
 
 func isDigit(r rune) bool {
-	return unicode.IsNumber(r)
+	return '0' <= r && r <= '9'
 }
 
 func isHexDigit(r rune) bool {

token.go

@@ -1,9 +1,6 @@
 package toml
 
-import (
-	"fmt"
-	"unicode"
-)
+import "fmt"
 
 // Define tokens
 type tokenType int
@@ -112,7 +109,7 @@ func isSpace(r rune) bool {
 }
 
 func isAlphanumeric(r rune) bool {
-	return unicode.IsLetter(r) || r == '_'
+	return 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || r == '_'
 }
 
 func isKeyChar(r rune) bool {
@@ -127,7 +124,7 @@ func isKeyStartChar(r rune) bool {
 }
 
 func isDigit(r rune) bool {
-	return unicode.IsNumber(r)
+	return '0' <= r && r <= '9'
 }
 
 func isHexDigit(r rune) bool {

tomltree_write.go

@@ -338,7 +338,7 @@ func (t *Tree) writeToOrdered(w io.Writer, indent, keyspace string, bytesCount i
 			k := node.key
 			v := t.values[k]
 
-			combinedKey := k
+			combinedKey := quoteKeyIfNeeded(k)
 			if keyspace != "" {
 				combinedKey = keyspace + "." + combinedKey
 			}