go ijson implementation (tested), ts ijson implemented (untested)

This commit is contained in:
sawka 2024-04-12 01:18:05 -07:00
parent d923de412a
commit 0fdcd27fee
3 changed files with 834 additions and 0 deletions

158
src/util/ijson.ts Normal file
View File

@ -0,0 +1,158 @@
// Copyright 2024, Command Line Inc.
// SPDX-License-Identifier: Apache-2.0
// ijson values are regular JSON values: string, number, boolean, null, object, array
// path is an array of strings and numbers
type PathType = (string | number)[];
var simplePathStrRe = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
function formatPath(path: PathType): string {
if (path.length == 0) {
return "$";
}
let pathStr = "$";
for (let pathPart of path) {
if (typeof pathPart === "string") {
if (simplePathStrRe.test(pathPart)) {
pathStr += "." + pathPart;
} else {
pathStr += "[" + JSON.stringify(pathPart) + "]";
}
} else {
pathStr += "[" + pathPart + "]";
}
}
return pathStr;
}
function getPath(obj: any, path: PathType): any {
let cur = obj;
for (let pathPart of path) {
if (cur == null) {
return null;
}
if (typeof pathPart === "string") {
if (cur instanceof Object) {
cur = cur[pathPart];
} else {
return null;
}
} else if (typeof pathPart === "number") {
if (Array.isArray(cur)) {
cur = cur[pathPart];
} else {
return null;
}
} else {
throw new Error("Invalid path part: " + pathPart);
}
}
return cur;
}
type SetPathOpts = {
force?: boolean;
remove?: boolean;
};
function setPath(obj: any, path: PathType, value: any, opts: SetPathOpts) {
if (opts == null) {
opts = {};
}
if (opts.remove && value != null) {
throw new Error("Cannot set value and remove at the same time");
}
setPathInternal(obj, path, value, opts);
}
function isEmpty(obj: any): boolean {
if (obj == null) {
return true;
}
if (obj instanceof Object) {
for (let key in obj) {
return false;
}
return true;
}
return false;
}
function removeFromArr(arr: any[], idx: number): any[] {
if (idx >= arr.length) {
return arr;
}
if (idx == arr.length - 1) {
arr.pop();
return arr;
}
arr[idx] = null;
return arr;
}
function setPathInternal(obj: any, path: PathType, value: any, opts: SetPathOpts): any {
if (path.length == 0) {
return value;
}
const pathPart = path[0];
if (typeof pathPart === "string") {
if (obj == null) {
if (opts.remove) {
return null;
}
obj = {};
}
if (!(obj instanceof Object)) {
if (opts.force) {
obj = {};
} else {
throw new Error("Cannot set path on non-object: " + obj);
}
}
if (opts.remove && path.length == 1) {
delete obj[pathPart];
if (isEmpty(obj)) {
return null;
}
return obj;
}
const newVal = setPath(obj[pathPart], path.slice(1), value, opts);
if (opts.remove && newVal == null) {
delete obj[pathPart];
if (isEmpty(obj)) {
return null;
}
return obj;
}
obj[pathPart] = newVal;
} else if (typeof pathPart === "number") {
if (pathPart < 0 || !Number.isInteger(pathPart)) {
throw new Error("Invalid path part: " + pathPart);
}
if (obj == null) {
if (opts.remove) {
return null;
}
obj = [];
}
if (!Array.isArray(obj)) {
if (opts.force) {
obj = [];
} else {
throw new Error("Cannot set path on non-array: " + obj);
}
}
if (opts.remove && path.length == 1) {
return removeFromArr(obj, pathPart);
}
const newVal = setPath(obj[pathPart], path.slice(1), value, opts);
if (opts.remove && newVal == null) {
return removeFromArr(obj, pathPart);
}
obj[pathPart] = newVal;
} else {
throw new Error("Invalid path part: " + pathPart);
}
}

View File

@ -0,0 +1,512 @@
// Copyright 2024, Command Line Inc.
// SPDX-License-Identifier: Apache-2.0
// implements incremental json format
package ijson
import (
"bytes"
"fmt"
"regexp"
"strconv"
)
// ijson values are built out of standard go building blocks:
// string, float64, bool, nil, []any, map[string]any
// paths are arrays of strings and ints
const (
SetCommandStr = "set"
DelCommandStr = "del"
AppendCommandStr = "append"
)
var _ = CommandType(SetCommand{})
var _ = CommandType(DelCommand{})
var _ = CommandType(AppendCommand{})
type CommandType interface {
GetCommandType() string
}
type SetCommand struct {
Type string `json:"type"` // "set"
Path []any `json:"path"` // each path entry must be either a string or int
Data any `json:"data"` // data must be a valid JSON structure
}
func (c SetCommand) GetCommandType() string {
return SetCommandStr
}
// removes values from the state
type DelCommand struct {
Type string `json:"type"` // "del"
Path []any `json:"path"`
}
func (c DelCommand) GetCommandType() string {
return DelCommandStr
}
// appends to an array
type AppendCommand struct {
Type string `json:"type"` // "append"
Path []any `json:"path"`
Data any `json:"data"`
}
func (c AppendCommand) GetCommandType() string {
return AppendCommandStr
}
type PathError struct {
Err string
}
func (e PathError) Error() string {
return "PathError: " + e.Err
}
func MakePathTypeError(path []any, index int) error {
return PathError{fmt.Sprintf("invalid path element type:%T at index:%d (%s)", path[index], index, FormatPath(path))}
}
func MakePathError(errStr string, path []any, index int) error {
return PathError{fmt.Sprintf("%s at index:%d (%s)", errStr, index, FormatPath(path))}
}
type SetTypeError struct {
Err string
}
func (e SetTypeError) Error() string {
return "SetTypeError: " + e.Err
}
func MakeSetTypeError(errStr string, path []any, index int) error {
return SetTypeError{fmt.Sprintf("%s at index:%d (%s)", errStr, index, FormatPath(path))}
}
type BudgetError struct {
Err string
}
func (e BudgetError) Error() string {
return "BudgetError: " + e.Err
}
func MakeBudgetError(errStr string, path []any, index int) error {
return BudgetError{fmt.Sprintf("%s at index:%d (%s)", errStr, index, FormatPath(path))}
}
var simplePathStrRe = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)
func FormatPath(path []any) string {
if len(path) == 0 {
return "$"
}
var buf bytes.Buffer
buf.WriteByte('$')
for _, elem := range path {
switch elem := elem.(type) {
case string:
if simplePathStrRe.MatchString(elem) {
buf.WriteByte('.')
buf.WriteString(elem)
} else {
buf.WriteByte('[')
buf.WriteString(strconv.Quote(elem))
buf.WriteByte(']')
}
case int:
buf.WriteByte('[')
buf.WriteString(strconv.Itoa(elem))
buf.WriteByte(']')
default:
// a placeholder for a bad value
buf.WriteString(".*")
}
}
return buf.String()
}
type pathWithPos struct {
Path []any
Index int
}
func (pp pathWithPos) isLast() bool {
return pp.Index == len(pp.Path)-1
}
func GetPath(data any, path []any) (any, error) {
return getPathInternal(data, pathWithPos{Path: path, Index: 0})
}
func getPathInternal(data any, pp pathWithPos) (any, error) {
if data == nil {
return nil, nil
}
if pp.Index >= len(pp.Path) {
return data, nil
}
pathElemAny := pp.Path[pp.Index]
switch pathElem := pathElemAny.(type) {
case string:
mapVal, ok := data.(map[string]any)
if !ok {
return nil, nil
}
return getPathInternal(mapVal[pathElem], pathWithPos{Path: pp.Path, Index: pp.Index + 1})
case int:
if pathElem < 0 {
return nil, MakePathError("negative index", pp.Path, pp.Index)
}
arrVal, ok := data.([]any)
if !ok {
return nil, nil
}
if pathElem >= len(arrVal) {
return nil, nil
}
return getPathInternal(arrVal[pathElem], pathWithPos{Path: pp.Path, Index: pp.Index + 1})
default:
return nil, MakePathTypeError(pp.Path, pp.Index)
}
}
type CombiningFunc func(curValue any, newValue any, pp pathWithPos, opts SetPathOpts) (any, error)
type SetPathOpts struct {
Budget int // Budget 0 is unlimited (to set a 0 value, use -1)
Force bool
Remove bool
CombineFn CombiningFunc
}
func SetPathNoErr(data any, path []any, value any, opts *SetPathOpts) any {
ret, _ := SetPath(data, path, value, opts)
return ret
}
func SetPath(data any, path []any, value any, opts *SetPathOpts) (any, error) {
if opts == nil {
opts = &SetPathOpts{}
}
if opts.Remove && opts.CombineFn != nil {
return nil, fmt.Errorf("SetPath: Remove and CombineFn are mutually exclusive")
}
if opts.Remove && value != nil {
return nil, fmt.Errorf("SetPath: Remove and value are mutually exclusive")
}
return setPathInternal(data, pathWithPos{Path: path, Index: 0}, value, *opts)
}
func checkAndModifyBudget(opts *SetPathOpts, pp pathWithPos, cost int) bool {
if opts.Budget == 0 {
return true
}
opts.Budget -= cost
if opts.Budget < 0 {
return false
}
if opts.Budget == 0 {
// 0 is weird since it means unlimited, so we set it to -1 to fail the next operation
opts.Budget = -1
}
return true
}
func CombineFn_ArrayAppend(data any, value any, pp pathWithPos, opts SetPathOpts) (any, error) {
if !checkAndModifyBudget(&opts, pp, 1) {
return nil, MakeBudgetError("trying to append to array", pp.Path, pp.Index)
}
if data == nil {
data = make([]any, 0)
}
arrVal, ok := data.([]any)
if !ok && !opts.Force {
return nil, MakeSetTypeError(fmt.Sprintf("expected array, but got %T", data), pp.Path, pp.Index)
}
if !ok {
arrVal = make([]any, 0)
}
arrVal = append(arrVal, value)
return arrVal, nil
}
func CombineFn_SetUnless(data any, value any, pp pathWithPos, opts SetPathOpts) (any, error) {
if data != nil {
return data, nil
}
return value, nil
}
func CombineFn_Max(data any, value any, pp pathWithPos, opts SetPathOpts) (any, error) {
valueFloat, ok := value.(float64)
if !ok {
return nil, MakeSetTypeError(fmt.Sprintf("expected float64, but got %T", value), pp.Path, pp.Index)
}
if data == nil {
return value, nil
}
dataFloat, ok := data.(float64)
if !ok && !opts.Force {
return nil, MakeSetTypeError(fmt.Sprintf("expected float64, but got %T", data), pp.Path, pp.Index)
}
if !ok {
return value, nil
}
if dataFloat > valueFloat {
return data, nil
}
return value, nil
}
func CombineFn_Min(data any, value any, pp pathWithPos, opts SetPathOpts) (any, error) {
valueFloat, ok := value.(float64)
if !ok {
return nil, MakeSetTypeError(fmt.Sprintf("expected float64, but got %T", value), pp.Path, pp.Index)
}
if data == nil {
return value, nil
}
dataFloat, ok := data.(float64)
if !ok && !opts.Force {
return nil, MakeSetTypeError(fmt.Sprintf("expected float64, but got %T", data), pp.Path, pp.Index)
}
if !ok {
return value, nil
}
if dataFloat < valueFloat {
return data, nil
}
return value, nil
}
func CombineFn_Inc(data any, value any, pp pathWithPos, opts SetPathOpts) (any, error) {
valueFloat, ok := value.(float64)
if !ok {
return nil, MakeSetTypeError(fmt.Sprintf("expected float64, but got %T", value), pp.Path, pp.Index)
}
if data == nil {
return value, nil
}
dataFloat, ok := data.(float64)
if !ok && !opts.Force {
return nil, MakeSetTypeError(fmt.Sprintf("expected float64, but got %T", data), pp.Path, pp.Index)
}
if !ok {
return value, nil
}
return dataFloat + valueFloat, nil
}
// force will clobber existing values that don't conform to path
// so SetPath(5, ["a"], 6 true) would return {"a": 6}
func setPathInternal(data any, pp pathWithPos, value any, opts SetPathOpts) (any, error) {
if pp.Index >= len(pp.Path) {
if opts.CombineFn != nil {
return opts.CombineFn(data, value, pp, opts)
}
return value, nil
}
pathElemAny := pp.Path[pp.Index]
switch pathElem := pathElemAny.(type) {
case string:
if data == nil {
if opts.Remove {
return nil, nil
}
data = make(map[string]any)
}
mapVal, ok := data.(map[string]any)
if !ok && !opts.Force {
return nil, MakeSetTypeError(fmt.Sprintf("expected map, but got %T", data), pp.Path, pp.Index)
}
if !ok {
mapVal = make(map[string]any)
}
if opts.Remove && pp.isLast() {
delete(mapVal, pathElem)
if len(mapVal) == 0 {
return nil, nil
}
return mapVal, nil
}
if _, ok := mapVal[pathElem]; !ok {
if opts.Remove {
return mapVal, nil
}
if !checkAndModifyBudget(&opts, pp, 1) {
return nil, MakeBudgetError("trying to allocate map entry", pp.Path, pp.Index)
}
}
newVal, err := setPathInternal(mapVal[pathElem], pathWithPos{Path: pp.Path, Index: pp.Index + 1}, value, opts)
if opts.Remove && newVal == nil {
delete(mapVal, pathElem)
if len(mapVal) == 0 {
return nil, nil
}
return mapVal, nil
}
mapVal[pathElem] = newVal
return mapVal, err
case int:
if pathElem < 0 {
return nil, MakePathError("negative index", pp.Path, pp.Index)
}
if data == nil {
if opts.Remove {
return nil, nil
}
if !checkAndModifyBudget(&opts, pp, pathElem+1) {
return nil, MakeBudgetError(fmt.Sprintf("trying to allocate array with %d elements", pathElem+1), pp.Path, pp.Index)
}
data = make([]any, pathElem+1)
}
arrVal, ok := data.([]any)
if !ok && !opts.Force {
return nil, MakeSetTypeError(fmt.Sprintf("expected array, but got %T", data), pp.Path, pp.Index)
}
if !ok {
if opts.Remove {
return nil, nil
}
if !checkAndModifyBudget(&opts, pp, pathElem+1) {
return nil, MakeBudgetError(fmt.Sprintf("trying to allocate array with %d elements", pathElem+1), pp.Path, pp.Index)
}
arrVal = make([]any, pathElem+1)
}
if opts.Remove && pp.isLast() {
if pathElem == len(arrVal)-1 {
arrVal = arrVal[:pathElem]
if len(arrVal) == 0 {
return nil, nil
}
return arrVal, nil
}
arrVal[pathElem] = nil
return arrVal, nil
}
entriesToAdd := pathElem + 1 - len(arrVal)
if opts.Remove && entriesToAdd > 0 {
return nil, nil
}
if !checkAndModifyBudget(&opts, pp, entriesToAdd) {
return nil, MakeBudgetError(fmt.Sprintf("trying to add %d elements to array", entriesToAdd), pp.Path, pp.Index)
}
for len(arrVal) <= pathElem {
arrVal = append(arrVal, nil)
}
newVal, err := setPathInternal(arrVal[pathElem], pathWithPos{Path: pp.Path, Index: pp.Index + 1}, value, opts)
if opts.Remove && newVal == nil && pathElem == len(arrVal)-1 {
arrVal = arrVal[:pathElem]
if len(arrVal) == 0 {
return nil, nil
}
return arrVal, nil
}
arrVal[pathElem] = newVal
return arrVal, err
default:
return nil, PathError{fmt.Sprintf("invalid path element type %T", pathElem)}
}
}
func NormalizeNumbers(v any) any {
switch v := v.(type) {
case int:
return float64(v)
case float32:
return float64(v)
case int8:
return float64(v)
case int16:
return float64(v)
case int32:
return float64(v)
case int64:
return float64(v)
case uint:
return float64(v)
case uint8:
return float64(v)
case uint16:
return float64(v)
case uint32:
return float64(v)
case uint64:
return float64(v)
case []any:
for i, elem := range v {
v[i] = NormalizeNumbers(elem)
}
case map[string]any:
for k, elem := range v {
v[k] = NormalizeNumbers(elem)
}
}
return v
}
func DeepEqual(v1 any, v2 any) bool {
if v1 == nil && v2 == nil {
return true
}
if v1 == nil || v2 == nil {
return false
}
switch v1 := v1.(type) {
case bool:
v2, ok := v2.(bool)
return ok && v1 == v2
case float64:
v2, ok := v2.(float64)
return ok && v1 == v2
case string:
v2, ok := v2.(string)
return ok && v1 == v2
case []any:
v2, ok := v2.([]any)
if !ok || len(v1) != len(v2) {
return false
}
for i := range v1 {
if !DeepEqual(v1[i], v2[i]) {
return false
}
}
return true
case map[string]any:
v2, ok := v2.(map[string]any)
if !ok || len(v1) != len(v2) {
return false
}
for k, v := range v1 {
if !DeepEqual(v, v2[k]) {
return false
}
}
return true
default:
// invalid data type, so just return false
return false
}
}
func ApplyCommand(data any, command CommandType, budget int) (any, error) {
switch command := command.(type) {
case SetCommand:
return SetPath(data, command.Path, command.Data, &SetPathOpts{Budget: budget})
case DelCommand:
return SetPath(data, command.Path, nil, &SetPathOpts{Remove: true, Budget: budget})
case AppendCommand:
return SetPath(data, command.Path, command.Data, &SetPathOpts{CombineFn: CombineFn_ArrayAppend, Budget: budget})
default:
return nil, fmt.Errorf("unknown command type %T", command)
}
}

View File

@ -0,0 +1,164 @@
// Copyright 2024, Command Line Inc.
// SPDX-License-Identifier: Apache-2.0
package ijson
import "testing"
func TestDeepEqual(t *testing.T) {
if !DeepEqual(float64(1), float64(1)) {
t.Errorf("DeepEqual(1, 1) should be true")
}
if DeepEqual(float64(1), float64(2)) {
t.Errorf("DeepEqual(1, 2) should be false")
}
if !DeepEqual([]any{"a", 2.8, true, map[string]any{"c": 1.1}}, []any{"a", 2.8, true, map[string]any{"c": 1.1}}) {
t.Errorf("DeepEqual complex should be true")
}
}
func TestGetPath(t *testing.T) {
data := []any{"a", 2.8, true, map[string]any{"c": 1.1}}
rtn, err := GetPath(data, []any{0})
if err != nil {
t.Errorf("GetPath failed: %v", err)
}
if rtn != "a" {
t.Errorf("GetPath failed: %v", rtn)
}
rtn, err = GetPath(data, []any{50})
if err != nil {
t.Errorf("GetPath failed: %v", err)
}
if rtn != nil {
t.Errorf("GetPath failed: %v", rtn)
}
rtn, err = GetPath(data, []any{3, "c"})
if err != nil {
t.Errorf("GetPath failed: %v", err)
}
if rtn != 1.1 {
t.Errorf("GetPath failed: %v", rtn)
}
}
func makeValue() any {
return []any{"a", 2.8, true, map[string]any{"c": 1.1}}
}
func TestSetPath(t *testing.T) {
rtn, err := SetPath(makeValue(), []any{0}, "b", nil)
if err != nil {
t.Errorf("SetPath failed: %v", err)
}
if rtn.([]any)[0] != "b" {
t.Errorf("SetPath failed: %v", rtn)
}
rtn, err = SetPath(makeValue(), []any{10}, "b", nil)
if err != nil {
t.Errorf("SetPath failed: %v", err)
}
if len(rtn.([]any)) != 11 {
t.Errorf("SetPath failed: %v", rtn)
}
rtn, _ = GetPath(rtn, []any{10})
if rtn != "b" {
t.Errorf("SetPath failed: %v", rtn)
}
_, err = SetPath(makeValue(), []any{"a"}, "b", nil)
if err == nil {
t.Errorf("SetPath should have failed")
}
rtn, err = SetPath(makeValue(), []any{"a"}, "b", &SetPathOpts{Force: true})
if err != nil {
t.Errorf("SetPath failed: %v", err)
}
if !DeepEqual(rtn, map[string]any{"a": "b"}) {
t.Errorf("SetPath failed: %v", rtn)
}
rtn, err = SetPath(makeValue(), nil, "c", &SetPathOpts{CombineFn: CombineFn_ArrayAppend})
if err != nil {
t.Errorf("SetPath failed: %v", err)
}
if !DeepEqual(rtn, []any{"a", 2.8, true, map[string]any{"c": 1.1}, "c"}) {
t.Errorf("SetPath failed: %v", rtn)
}
_, err = SetPath(makeValue(), nil, "c", &SetPathOpts{CombineFn: CombineFn_ArrayAppend, Budget: -1})
if err == nil {
t.Errorf("SetPath should have failed")
}
rtn, err = SetPath(makeValue(), []any{5000}, "c", nil)
if err != nil {
t.Errorf("SetPath failed: %v", err)
}
if len(rtn.([]any)) != 5001 {
t.Errorf("SetPath failed: %v", rtn)
}
_, err = SetPath(makeValue(), []any{5000}, "c", &SetPathOpts{Budget: 1000})
if err == nil {
t.Errorf("SetPath should have failed")
}
rtn, err = SetPath(makeValue(), []any{3, "c"}, nil, &SetPathOpts{Remove: true})
if err != nil {
t.Errorf("SetPath failed: %v", err)
}
if !DeepEqual(rtn, []any{"a", 2.8, true}) {
t.Errorf("SetPath failed: %v", rtn)
}
rtn, _ = SetPath(makeValue(), []any{3}, nil, &SetPathOpts{Remove: true})
rtn, _ = SetPath(rtn, []any{2}, nil, &SetPathOpts{Remove: true})
rtn, _ = SetPath(rtn, []any{1}, nil, &SetPathOpts{Remove: true})
rtn, _ = SetPath(rtn, []any{0}, nil, &SetPathOpts{Remove: true})
if rtn != nil {
t.Errorf("SetPath failed: %v", rtn)
}
rtn, err = SetPath(makeValue(), []any{3, "d"}, 2.2, nil)
if err != nil {
t.Errorf("SetPath failed: %v", err)
}
if !DeepEqual(rtn, []any{"a", 2.8, true, map[string]any{"c": 1.1, "d": 2.2}}) {
t.Errorf("SetPath failed: %v", rtn)
}
rtn, err = SetPath(makeValue(), []any{1}, 2.2, &SetPathOpts{CombineFn: CombineFn_Inc})
if err != nil {
t.Errorf("SetPath failed: %v", err)
}
if !DeepEqual(rtn, []any{"a", 5.0, true, map[string]any{"c": 1.1}}) {
t.Errorf("SetPath failed: %v", rtn)
}
rtn, err = SetPath(makeValue(), []any{1}, 500.0, &SetPathOpts{CombineFn: CombineFn_Min})
if err != nil {
t.Errorf("SetPath failed: %v", err)
}
if rtn.([]any)[1] != 2.8 {
t.Errorf("SetPath failed: %v", rtn)
}
rtn, err = SetPath(makeValue(), []any{1}, 500.0, &SetPathOpts{CombineFn: CombineFn_Max})
if err != nil {
t.Errorf("SetPath failed: %v", err)
}
if rtn.([]any)[1] != 500.0 {
t.Errorf("SetPath failed: %v", rtn)
}
rtn, err = SetPath(makeValue(), []any{1}, 500.0, &SetPathOpts{CombineFn: CombineFn_SetUnless})
if err != nil {
t.Errorf("SetPath failed: %v", err)
}
if rtn.([]any)[1] != 2.8 {
t.Errorf("SetPath failed: %v", rtn)
}
rtn, err = SetPath(makeValue(), []any{8}, 500.0, &SetPathOpts{CombineFn: CombineFn_SetUnless})
if err != nil {
t.Errorf("SetPath failed: %v", err)
}
if rtn.([]any)[8] != 500.0 {
t.Errorf("SetPath failed: %v", rtn)
}
}