Merge pull request #203 from moonD4rk/fix/leveldb

fix: missing key and value in local storage
pull/209/head
ᴍᴏᴏɴD4ʀᴋ 2 years ago committed by GitHub
commit 242f351fd0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 29
      browingdata/localstorage/localstorage.go
  2. 33
      browingdata/localstorage/localstorage_test.go
  3. 4
      go.mod
  4. 5
      go.sum
  5. 10
      utils/byteutil/byteutil.go

@ -8,9 +8,12 @@ import (
"strings"
"github.com/syndtr/goleveldb/leveldb"
"golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
"github.com/moond4rk/HackBrowserData/item"
"github.com/moond4rk/HackBrowserData/log"
"github.com/moond4rk/HackBrowserData/utils/byteutil"
"github.com/moond4rk/HackBrowserData/utils/typeutil"
)
@ -23,6 +26,8 @@ type storage struct {
Value string
}
const maxLocalStorageValueLength = 1024 * 2
func (c *ChromiumLocalStorage) Parse(masterKey []byte) error {
db, err := leveldb.OpenFile(item.TempChromiumLocalStorage, nil)
if err != nil {
@ -35,16 +40,16 @@ func (c *ChromiumLocalStorage) Parse(masterKey []byte) error {
for iter.Next() {
key := iter.Key()
value := iter.Value()
// don't parse value upper than 5kB
if len(value) > 1024*5 {
continue
}
s := new(storage)
s.fillKey(key)
s.fillValue(value)
// don't save meta data
// don't all value upper than 2KB
if len(value) < maxLocalStorageValueLength {
s.fillValue(value)
} else {
s.Value = fmt.Sprintf("value is too long, length is %d, supportted max length is %d", len(value), maxLocalStorageValueLength)
}
if s.IsMeta {
continue
s.Value = fmt.Sprintf("meta data, value bytes is %v", value)
}
*c = append(*c, *s)
}
@ -81,12 +86,16 @@ func (s *storage) fillHeader(url, key []byte) {
s.Key = string(bytes.Trim(key, "\x01"))
}
func convertUTF16toUTF8(source []byte, endian unicode.Endianness) ([]byte, error) {
r, _, err := transform.Bytes(unicode.UTF16(endian, unicode.IgnoreBOM).NewDecoder(), source)
return r, err
}
// fillValue fills value of the storage
// TODO: support unicode charter
func (s *storage) fillValue(b []byte) {
t := fmt.Sprintf("%c", b)
m := strings.NewReplacer(" ", "", "\x00", "", "\x01", "").Replace(t)
s.Value = m
value := bytes.Map(byteutil.OnSplitUTF8Func, b)
s.Value = string(value)
}
type FirefoxLocalStorage []storage

@ -0,0 +1,33 @@
package localstorage
import (
"testing"
"github.com/stretchr/testify/assert"
"golang.org/x/text/encoding/unicode"
)
var testCases = []struct {
in []byte
wanted []byte
actual []byte
}{
{
in: []byte{0x0, 0x7b, 0x0, 0x22, 0x0, 0x72, 0x0, 0x65, 0x0, 0x66, 0x0, 0x65, 0x0, 0x72, 0x0, 0x5f, 0x0, 0x6b, 0x0, 0x65, 0x0, 0x79, 0x0, 0x22, 0x0, 0x3a, 0x0, 0x22, 0x0, 0x68, 0x0, 0x74, 0x0, 0x74, 0x0, 0x70, 0x0, 0x73, 0x0, 0x3a, 0x0, 0x2f, 0x0, 0x2f, 0x0, 0x77, 0x0, 0x77, 0x0, 0x77, 0x0, 0x2e, 0x0, 0x76, 0x0, 0x6f, 0x0, 0x6c, 0x0, 0x63, 0x0, 0x65, 0x0, 0x6e, 0x0, 0x67, 0x0, 0x69, 0x0, 0x6e, 0x0, 0x65, 0x0, 0x2e, 0x0, 0x63, 0x0, 0x6f, 0x0, 0x6d, 0x0, 0x2f, 0x0, 0x70, 0x0, 0x72, 0x0, 0x6f, 0x0, 0x64, 0x0, 0x75, 0x0, 0x63, 0x0, 0x74, 0x0, 0x73, 0x0, 0x2f, 0x0, 0x66, 0x0, 0x65, 0x0, 0x69, 0x0, 0x6c, 0x0, 0x69, 0x0, 0x61, 0x0, 0x6e, 0x0, 0x22, 0x0, 0x2c, 0x0, 0x22, 0x0, 0x72, 0x0, 0x65, 0x0, 0x66, 0x0, 0x65, 0x0, 0x72, 0x0, 0x5f, 0x0, 0x74, 0x0, 0x69, 0x0, 0x74, 0x0, 0x6c, 0x0, 0x65, 0x0, 0x22, 0x0, 0x3a, 0x0, 0x22, 0x0, 0xde, 0x98, 0xde, 0x8f, 0x2d, 0x0, 0x6b, 0x70, 0x71, 0x5c, 0x15, 0x5f, 0xce, 0x64, 0x22, 0x0, 0x2c, 0x0, 0x22, 0x0, 0x72, 0x0, 0x65, 0x0, 0x66, 0x0, 0x65, 0x0, 0x72, 0x0, 0x5f, 0x0, 0x6d, 0x0, 0x61, 0x0, 0x6e, 0x0, 0x75, 0x0, 0x61, 0x0, 0x6c, 0x0, 0x5f, 0x0, 0x6b, 0x0, 0x65, 0x0, 0x79, 0x0, 0x22, 0x0, 0x3a, 0x0, 0x22, 0x0, 0x22, 0x0, 0x7d, 0x0},
wanted: []byte(`{"refer_key":"https://www.volcengine.com/product/feilian","refer_title":"飞连_SSO单点登录_VPN_终端安全合规_便捷Wifi认证-火山引擎","refer_manual_key":""}`),
actual: []byte{0x7b, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x3a, 0x22, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x6f, 0x6c, 0x63, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x73, 0x2f, 0x66, 0x65, 0x69, 0x6c, 0x69, 0x61, 0x6e, 0x22, 0x2c, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x5f, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x22, 0x3a, 0x22, 0xc3, 0x9e, 0xe9, 0xa3, 0x9e, 0xe8, 0xbc, 0xad, 0x6b, 0xe7, 0x81, 0xb1, 0xe5, 0xb0, 0x95, 0xe5, 0xbf, 0x8e, 0xe6, 0x90, 0xa2, 0x2c, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x3a, 0x22, 0x22, 0x7d, 0xef, 0xbf, 0xbd},
},
}
func TestLocalStorageKeyToUTF8(t *testing.T) {
t.Parallel()
for _, tc := range testCases {
actual, err := convertUTF16toUTF8(tc.in, unicode.BigEndian)
if err != nil {
t.Error(err)
}
// TODO: fix this, value from local storage if contains chinese characters, need convert utf16 to utf8
// but now, it can't convert, so just skip it.
assert.Equal(t, tc.actual, actual, "chinese characters can't actual convert")
}
}

@ -9,6 +9,7 @@ require (
github.com/mattn/go-sqlite3 v1.14.16
github.com/otiai10/copy v1.9.0
github.com/ppacher/go-dbus-keyring v1.0.1
github.com/stretchr/testify v1.8.2
github.com/syndtr/goleveldb v1.0.0
github.com/tidwall/gjson v1.14.4
github.com/urfave/cli/v2 v2.25.0
@ -19,10 +20,12 @@ require (
require (
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
github.com/gookit/color v1.5.2 // indirect
github.com/gookit/goutil v0.6.1 // indirect
github.com/gookit/gsr v0.0.8 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
@ -30,4 +33,5 @@ require (
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/sys v0.6.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

@ -44,9 +44,11 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
@ -79,6 +81,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=

@ -0,0 +1,10 @@
package byteutil
var (
OnSplitUTF8Func = func(r rune) rune {
if r == 0x00 || r == 0x01 {
return -1
}
return r
}
)
Loading…
Cancel
Save