Working on JSON-based rewrite

This commit is contained in:
Brian Buller 2024-11-07 12:50:11 -06:00
parent 319a3b033f
commit 741eac472a
3 changed files with 572 additions and 4 deletions

560
be_encode.go Normal file
View File

@ -0,0 +1,560 @@
package boltease
import (
"encoding"
"fmt"
"math"
"reflect"
"slices"
"strconv"
"strings"
"sync"
)
func (db *DB) Save(path []string, v any) error {
e := newWriterState(db, path)
defer writerStatePool.Put(e)
err := e.marshal(db, path, v, encOpts{})
if err != nil {
return err
}
return nil
}
// Marshaler is the interfacce implemented by types that
// can marshal themselves into a db
type Marshaler interface {
MarshalBoltease(db *DB, path []string) error
}
// An UnsupportedTypeError is returned by [Marsha] when attempting
// to encode an unsupported value type.
type UnsupportedTypeError struct {
Type reflect.Type
}
func (e *UnsupportedTypeError) Error() string {
return "boltease: unsupported type: " + e.Type.String()
}
// A MarshalError represents an error from calling a
// [Marshaler.MarshelBoltease] or [encoding.TextMarshaler.MarshalText] method.
type MarshalerError struct {
Type reflect.Type
Err error
sourceFunc string
}
func (e *MarshalerError) Error() string {
srcFunc := e.sourceFunc
if srcFunc == "" {
srcFunc = "MarshalBoltease"
}
return "boltease: error calling " + srcFunc +
" for type " + e.Type.String() +
": " + e.Err.Error()
}
// Unwrap returns the underlying error.
func (e *MarshalerError) Unwrap() error { return e.Err }
type writerState struct {
db *DB
path []string
ptrLevel uint
ptrSeen map[any]struct{}
}
func (es *writerState) WriteString(val string) error {
return es.Write(key, []byte(val))
}
func (es *writerState) Write(key []byte, val []byte) error {
return es.db.SetBBytes(es.path, key, val)
}
const startDetectingCyclesAfter = 1000
var writerStatePool sync.Pool
func newWriterState(db *DB, path []string) *writerState {
if v := writerStatePool.Get(); v != nil {
e := v.(*writerState)
if len(e.ptrSeen) > 0 {
panic("ptrEncoder.encode should have emptied ptrSeen via defers")
}
e.ptrLevel = 0
return e
}
return &writerState{
db: db,
path: path,
ptrSeen: make(map[any]struct{}),
}
}
// bolteaseError is an error wrapper type for internal use only.
// Panics with errors are wrapped in bolteaseError so that the top-level recover
// can distinguish intentional panics from this package.
type bolteaseError struct{ error }
func (e *writerState) marshal(db *DB, path []string, v any, opts writerOpts) (err error) {
defer func() {
if r := recover(); r != nil {
if be, ok := r.(bolteaseError); ok {
err = be.error
} else {
panic(r)
}
}
}()
e.reflectValue(reflect.ValueOf(v), opts)
return nil
}
// error aborts the encoding by panicking with err wrapped in bolteaseError.
func (e *writerState) error(err error) {
panic(bolteaseError{err})
}
func isEmptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool,
reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
reflect.Float32, reflect.Float64,
reflect.Interface, reflect.Pointer:
return v.IsZero()
}
return false
}
func (e *writerState) reflectValue(v reflect.Value, opts writerOpts) {
valueWriter(v)(e, v, opts)
}
type writerOpts struct {
// quoted causes primitive fields to be encoded inside Boltease strings.
quoted bool
}
type writerFunc func(e *writerState, v reflect.Value, opts writerOpts)
var writerCache sync.Map // map[reflect.Type]writerFunc
func valueWriter(v reflect.Value) writerFunc {
if !v.IsValid() {
return invalidValueWriter
}
return typeWriter(v.Type())
}
func typeWriter(t reflect.Type) writerFunc {
if fi, ok := writerCache.Load(t); ok {
return fi.(writerFunc)
}
// To deal with recursive types, populate the map with an
// indirect func before we build it. This type waits on the
// real func (f) to be ready and then calls it. This indirect
// func is only used for recursive types.
var (
wg sync.WaitGroup
f writerFunc
)
wg.Add(1)
fi, loaded := writerCache.LoadOrStore(t, writerFunc(func(e *writerState, v reflect.Value, opts writerOpts) {
wg.Wait()
f(e, v, opts)
}))
if loaded {
return fi.(writerFunc)
}
// Compute the real encoder and replace the indirect func with it.
f = newTypeWriter(t, true)
wg.Done()
writerCache.Store(t, f)
return f
}
var (
marshalerType = reflect.TypeFor[Marshaler]()
textMarshalerType = reflect.TypeFor[encoding.TextMarshaler]()
)
// newTypeEncoder constructs an encoderFunc for a type.
// The returned encoder only checks CanAddr when allowAddr is true.
func newTypeEncoder(t reflect.Type, allowAddr bool) encoderFunc {
// if we have a non-pointer value whose type implements
// Marshaler with a value receiver, then we're better off taking
// the address of the value - otherwise we end up with an
// allocation as we cast the value to an interface.
if t.Kind() != reflect.Pointer && allowAddr && reflect.PointerTo(t).Implements(marshalerType) {
return newCondAddrEncoder(addrMarshalerEncoder, newTypeEncoder(t, false))
}
if t.Implements(marshalerType) {
return marshalerEncoder
}
if t.Kind() != reflect.Pointer && allowAddr && reflect.PointerTo(t).Implements(textMarshalerType) {
return newCondAddrEncoder(addrTextMarshalerEncoder, newTypeEncoder(t, false))
}
if t.Implements(textMarshalerType) {
return textMarshalerEncoder
}
switch t.Kind() {
case reflect.Bool:
return boolEncoder
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return intEncoder
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return uintEncoder
case reflect.Float32:
return float32Encoder
case reflect.Float64:
return float64Encoder
case reflect.String:
return stringEncoder
case reflect.Interface:
return interfaceEncoder
case reflect.Struct:
return newStructEncoder(t)
case reflect.Map:
return newMapEncoder(t)
case reflect.Slice:
return newSliceEncoder(t)
case reflect.Array:
return newArrayEncoder(t)
case reflect.Pointer:
return newPtrEncoder(t)
default:
return unsupportedTypeEncoder
}
}
func invalidValueEncoder(e *encodeState, v reflect.Value, _ encOpts) {
e.WriteString("null")
}
func marshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) {
if v.Kind() == reflect.Pointer && v.IsNil() {
e.WriteString("null")
return
}
m, ok := v.Interface().(Marshaler)
if !ok {
e.WriteString("null")
return
}
err := m.MarshalBoltease(e.db, e.path)
if err != nil {
e.error(&MarshalerError{v.Type(), err, "MarshalBoltease"})
}
}
func addrMarshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) {
va := v.Addr()
if va.IsNil() {
e.WriteString("null")
return
}
m := va.Interface().(Marshaler)
err := m.MarshalBoltease()
if err != nil {
e.error(&MarshalerError{v.Type(), err, "MarshalBoltease"})
}
}
func textMarshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) {
if v.Kind() == reflect.Pointer && v.IsNil() {
e.WriteString("null")
return
}
m, ok := v.Interface().(encoding.TextMarshaler)
if !ok {
e.WriteString("null")
return
}
b, err := m.MarshalText()
if err != nil {
e.error(&MarshalerError{v.Type(), err, "MarshalText"})
}
e.Write(b)
}
func boolEncoder(e *encodeState, v reflect.Value, opts encOpts) {
b := []byte{}
b = mayAppendQuote(b, opts.quoted)
b = strconv.AppendBool(b, v.Bool())
b = mayAppendQuote(b, opts.quoted)
e.Write(b)
}
func intEncoder(e *encodeState, v reflect.Value, opts encOpts) {
b := []byte{}
b = mayAppendQuote(b, opts.quoted)
b = strconv.AppendInt(b, v.Int(), 10)
b = mayAppendQuote(b, opts.quoted)
e.Write(b)
}
func uintEncoder(e *encodeState, v reflect.Value, opts encOpts) {
b := []byte{}
b = mayAppendQuote(b, opts.quoted)
b = strconv.AppendUint(b, v.Uint(), 10)
b = mayAppendQuote(b, opts.quoted)
e.Write(b)
}
type floatEncoder int // number of bits
func (bits floatEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) {
f := v.Float()
if math.IsInf(f, 0) || math.IsNaN(f) {
e.error(&UnsupportedValueError{v, strconv.FormatFloat(f, 'g', -1, int(bits))})
}
// Connvert as if by ES6 number to string conversion.
// This matches most other JSON generators.
// See golang.org/issue/6384 and golang.org/issue/14135.
// Like fmt %g, but the exponent cutoffs are different
// and exponents themselves are not padded to two digits.
b := []byte{}
b = mayAppendQuote(b, opts.quoted)
abs := math.Abs(f)
fmt := byte('f')
// Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right.
if abs != 0 {
if bits == 64 && (abs < 1e-6 || abs >= 1e21) || bits == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) {
fmt = 'e'
}
}
b = strconv.AppendFloat(b, f, fmt, -1, int(bits))
if fmt == 'e' {
// clean up e-09 to e-9
n := len(b)
if n >= 4 && b[n-4] == 'e' && b[n-3] == '-' && b[n-2] == '0' {
b[n-2] = b[n-1]
b = b[:n-1]
}
}
b = mayAppendQuote(b, opts.quoted)
e.Write(b)
}
var (
float32Encoder = (floatEncoder(32)).encode
float64Encoder = (floatEncoder(64)).encode
)
func stringEncoder(e *encodeState, v reflect.Value, opts encOpts) {
if v.Type() == numberType {
numStr := v.String()
// In Go1.5 the empty string encodes to "0", while this is not a valid number literal
// we keep compatibility so check validity after this.
if numStr == "" {
numStr = "0" // Number's zero-val
}
if !isValidNumber(numStr) {
e.error(fmt.Errorf("json: invalid number literal %q", numStr))
}
b := []byte{}
b = mayAppendQuote(b, opts.quoted)
b = append(b, numStr...)
b = mayAppendQuote(b, opts.quoted)
e.Write(b)
return
}
if opts.quoted {
b := appendString(nil, v.String(), opts.escapeHTML)
e.Write(b) // no need to escape again since it is already escaped
} else {
e.Write([]byte(v.String()))
}
}
func isValidNumber(s string) bool {
// This function implements the JSON numbers grammar.
// See https://tools.ietf.org/html/rfc7159#section-6
// and https://www.json.org/img/number.png
if s == "" {
return false
}
// Optional -
if s[0] == '-' {
s = s[1:]
if s == "" {
return false
}
}
// Digits
switch {
default:
return false
case s[0] == '0':
s = s[1:]
case '1' <= s[0] && s[0] <= '9':
s = s[1:]
for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
s = s[1:]
}
}
// . followed by 1 or more digits.
if len(s) >= 2 && s[0] == '.' && '0' <= s[1] && s[1] <= '9' {
s = s[2:]
for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
s = s[1:]
}
}
// e or E followed by an optional - or + and
// 1 or more digits.
if len(s) >= 2 && (s[0] == 'e' || s[0] == 'E') {
s = s[1:]
if s[0] == '+' || s[0] == '-' {
s = s[1:]
if s == "" {
return false
}
}
for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
s = s[1:]
}
}
// Make sure we are at the end.
return s == ""
}
func interfaceEncoder(e *encodeState, v reflect.Value, opts encOpts) {
if v.IsNil() {
e.WriteString("null")
return
}
e.reflectValue(v.Elem(), opts)
}
func unsupportedTypeEncoder(e *encodeState, v reflect.Value, _ encOpts) {
e.error(&UnsupportedTypeError{v.Type()})
}
type structEncoder struct {
fields structFields
}
type structFields struct {
list []field
byExactName map[string]*field
byFoldedName map[string]*field
}
func (se structEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) {
FieldLoop:
for i := range se.fields.list {
f := &se.fields.list[i]
// Find the nested struct field by following f.index.
fv := v
for _, i := range f.index {
if fv.Kind() == reflect.Pointer {
if fv.IsNil() {
continue FieldLoop
}
fv = fv.Elem()
}
fv = fv.Field(i)
}
if f.omitEmpty && isEmptyValue(fv) {
continue
}
opts.quoted = f.quoted
f.encoder(e, fv, opts)
}
}
func newStrucEncoder(t reflect.Type) encoderFunc {
se := structEncoder{fields: cachedTypeFields(t)}
return se.encode
}
type mapEncoder struct {
elemEnc encoderFunc
}
func (me mapEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) {
if v.IsNil() {
e.WriteString("null")
return
}
if e.ptrLevel++; e.ptrLevel > startDetectingCyclesAfter {
// We're a large number of nested ptrEncoder.encode calls deep;
// start checking if we've run into a pointer cycle.
ptr := v.UnsafePointer()
if _, ok := e.ptrSeen[ptr]; ok {
e.error(&UnsupportedValueError{v, fmt.Sprintf("encountered a cycle via %s", v.Type())})
}
e.ptrSeen[ptr] = struct{}{}
defer delete(e.ptrSeen, ptr)
}
// Extract and sort the keys.
var (
sv = make([]reflectWithString, v.Len())
mi = v.MapRange()
err error
)
for i := 0; mi.Next(); i++ {
if sv[i].ks, err = resolveKeyName(mi.Key()); err != nil {
e.error(fmt.Errorf("boltease: encoding error for type %q: %q", v.Type().String(), err.Error()))
}
sv[i].v = mi.Value()
}
slices.SortFunc(sv, func(i, j reflectWithString) int {
return strings.Compare(i.ks, j.ks)
})
for i, kv := range sv {
kv.ks, me.elemEnc(e, kv.v, opts))
}
e.ptrLevel--
}
func newMapEncoder(t reflect.Type) encoderFunc {
switch t.Key().Kind() {
case reflect.String,
reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
default:
if !t.Key().Implements(textMarshalerType) {
return unsupportedTypeEncoder
}
}
me := mapEncoder{typeEncoder(t.Elem())}
return me.encode
}
// TODO: HERE: json/encode.go:793
// A field represents a single field found in a struct.
type field struct {
name string
nameBytes []byte
tag bool
index []int
typ reflect.Type
omitEmpty bool
quoted bool
encoder encoderFunc
}

View File

@ -12,7 +12,7 @@ import (
type any = interface{}
func (b *DB) Save(path []string, src any) error {
func (b *DB) SaveOld(path []string, src any) error {
t := reflect.TypeOf(src)
if t.Kind() == reflect.Pointer {
// Save the actual struct
@ -49,7 +49,7 @@ func (b *DB) Save(path []string, src any) error {
return nil
}
func (b *DB) Load(path []string, dest any) error {
func (b *DB) LoadOld(path []string, dest any) error {
destValue := reflect.ValueOf(dest)
fmt.Println("Loading:", path, destValue)
if destValue.Kind() != reflect.Pointer {
@ -192,6 +192,7 @@ func FieldName(fld reflect.StructField) string {
}
return nm
}
func FieldIgnored(fld reflect.StructField) bool {
return fld.Tag.Get("boltease") == "-"
}

View File

@ -125,6 +125,7 @@ func (b *DB) Set(path []string, key string, val interface{}) error {
return fmt.Errorf("Set: Unknown Data Type: %v", v)
}
}
func (b *DB) CanGetForInterface(val interface{}) bool {
switch val.(type) {
case *string, *int, *int8, *int16, *int32, *int64, *bool, *[]byte:
@ -133,6 +134,7 @@ func (b *DB) CanGetForInterface(val interface{}) bool {
return false
}
}
func (b *DB) GetForInterface(path []string, key string, val interface{}) error {
var err error
switch v := val.(type) {
@ -165,6 +167,7 @@ func (b *DB) GetForInterface(path []string, key string, val interface{}) error {
}
return err
}
func (b *DB) LoadInto(path []string, key string, into interface{}) error {
var err error
switch v := into.(type) {
@ -229,7 +232,7 @@ func (b *DB) GetBytes(path []string, key string) ([]byte, error) {
return ret, nil
}
func (b *DB) SetBytes(path []string, key string, val []byte) error {
func (b *DB) SetBBytes(path []string, key []byte, val []byte) error {
var err error
if !b.dbIsOpen {
if err = b.OpenDB(); err != nil {
@ -254,11 +257,15 @@ func (b *DB) SetBytes(path []string, key string, val []byte) error {
}
}
// bkt should have the last bucket in the path
return bkt.Put([]byte(key), val)
return bkt.Put(key, val)
})
return err
}
func (b *DB) SetBytes(path []string, key string, val []byte) error {
return b.SetBBytes(path, []byte(key), val)
}
// GetString returns the value at path
// path is a slice of strings
// key is the key to get