feat: test case for convert utf16 value to utf8

browingdata/localstorage/localstorage.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 
 	"github.com/syndtr/goleveldb/leveldb"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 
 	"github.com/moond4rk/HackBrowserData/item"
 	"github.com/moond4rk/HackBrowserData/log"
@@ -44,7 +46,7 @@ func (c *ChromiumLocalStorage) Parse(masterKey []byte) error {
 		if len(value) < maxLocalStorageLength {
 			s.fillValue(value)
 		} else {
-			s.Value = fmt.Sprintf("value is too long, length is %d", len(value))
+			s.Value = fmt.Sprintf("value is too long, length is %d, supportted max length is %d", len(value), maxLocalStorageLength)
 		}
 		if s.IsMeta {
 			s.Value = fmt.Sprintf("meta data, value bytes is %v", value)
@@ -84,6 +86,11 @@ func (s *storage) fillHeader(url, key []byte) {
 	s.Key = string(bytes.Trim(key, "\x01"))
 }
 
+func convertUTF16toUTF8(source []byte, endian unicode.Endianness) ([]byte, error) {
+	r, _, err := transform.Bytes(unicode.UTF16(endian, unicode.IgnoreBOM).NewDecoder(), source)
+	return r, err
+}
+
 // fillValue fills value of the storage
 // TODO: support unicode charter
 func (s *storage) fillValue(b []byte) {

browingdata/localstorage/localstorage_test.go

@@ -0,0 +1,32 @@
+package localstorage
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/text/encoding/unicode"
+)
+
+var testCases = []struct {
+	in     []byte
+	wanted []byte
+	actual []byte
+}{
+	{
+		in:     []byte{0x0, 0x7b, 0x0, 0x22, 0x0, 0x72, 0x0, 0x65, 0x0, 0x66, 0x0, 0x65, 0x0, 0x72, 0x0, 0x5f, 0x0, 0x6b, 0x0, 0x65, 0x0, 0x79, 0x0, 0x22, 0x0, 0x3a, 0x0, 0x22, 0x0, 0x68, 0x0, 0x74, 0x0, 0x74, 0x0, 0x70, 0x0, 0x73, 0x0, 0x3a, 0x0, 0x2f, 0x0, 0x2f, 0x0, 0x77, 0x0, 0x77, 0x0, 0x77, 0x0, 0x2e, 0x0, 0x76, 0x0, 0x6f, 0x0, 0x6c, 0x0, 0x63, 0x0, 0x65, 0x0, 0x6e, 0x0, 0x67, 0x0, 0x69, 0x0, 0x6e, 0x0, 0x65, 0x0, 0x2e, 0x0, 0x63, 0x0, 0x6f, 0x0, 0x6d, 0x0, 0x2f, 0x0, 0x70, 0x0, 0x72, 0x0, 0x6f, 0x0, 0x64, 0x0, 0x75, 0x0, 0x63, 0x0, 0x74, 0x0, 0x73, 0x0, 0x2f, 0x0, 0x66, 0x0, 0x65, 0x0, 0x69, 0x0, 0x6c, 0x0, 0x69, 0x0, 0x61, 0x0, 0x6e, 0x0, 0x22, 0x0, 0x2c, 0x0, 0x22, 0x0, 0x72, 0x0, 0x65, 0x0, 0x66, 0x0, 0x65, 0x0, 0x72, 0x0, 0x5f, 0x0, 0x74, 0x0, 0x69, 0x0, 0x74, 0x0, 0x6c, 0x0, 0x65, 0x0, 0x22, 0x0, 0x3a, 0x0, 0x22, 0x0, 0xde, 0x98, 0xde, 0x8f, 0x2d, 0x0, 0x6b, 0x70, 0x71, 0x5c, 0x15, 0x5f, 0xce, 0x64, 0x22, 0x0, 0x2c, 0x0, 0x22, 0x0, 0x72, 0x0, 0x65, 0x0, 0x66, 0x0, 0x65, 0x0, 0x72, 0x0, 0x5f, 0x0, 0x6d, 0x0, 0x61, 0x0, 0x6e, 0x0, 0x75, 0x0, 0x61, 0x0, 0x6c, 0x0, 0x5f, 0x0, 0x6b, 0x0, 0x65, 0x0, 0x79, 0x0, 0x22, 0x0, 0x3a, 0x0, 0x22, 0x0, 0x22, 0x0, 0x7d, 0x0},
+		wanted: []byte(`{"refer_key":"https://www.volcengine.com/product/feilian","refer_title":"飞连_SSO单点登录_VPN_终端安全合规_便捷Wifi认证-火山引擎","refer_manual_key":""}`),
+		actual: []byte{0x7b, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x3a, 0x22, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x6f, 0x6c, 0x63, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x73, 0x2f, 0x66, 0x65, 0x69, 0x6c, 0x69, 0x61, 0x6e, 0x22, 0x2c, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x5f, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x22, 0x3a, 0x22, 0xc3, 0x9e, 0xe9, 0xa3, 0x9e, 0xe8, 0xbc, 0xad, 0x6b, 0xe7, 0x81, 0xb1, 0xe5, 0xb0, 0x95, 0xe5, 0xbf, 0x8e, 0xe6, 0x90, 0xa2, 0x2c, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x3a, 0x22, 0x22, 0x7d, 0xef, 0xbf, 0xbd},
+	},
+}
+
+func TestLocalStorageKeyToUTF8(t *testing.T) {
+	for _, tc := range testCases {
+		actual, err := convertUTF16toUTF8(tc.in, unicode.BigEndian)
+		if err != nil {
+			t.Error(err)
+		}
+		// TODO: fix this, value from local storage if contains chinese characters, need convert utf16 to utf8
+		// but now, it can't convert, so just skip it.
+		assert.Equal(t, tc.actual, actual, "chinese characters can't actual convert")
+	}
+}

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.16
 	github.com/otiai10/copy v1.9.0
 	github.com/ppacher/go-dbus-keyring v1.0.1
+	github.com/stretchr/testify v1.8.2
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/tidwall/gjson v1.14.4
 	github.com/urfave/cli/v2 v2.25.0
@@ -19,10 +20,12 @@ require (
 
 require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
 	github.com/gookit/color v1.5.2 // indirect
 	github.com/gookit/goutil v0.6.1 // indirect
 	github.com/gookit/gsr v0.0.8 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
@@ -30,4 +33,5 @@ require (
 	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	golang.org/x/sys v0.6.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -44,9 +44,11 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
@@ -79,6 +81,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=