checkpoint

pkg/binpack/binpack.go

@@ -2,6 +2,7 @@ package binpack
 
 import (
 	"encoding/binary"
+	"encoding/json"
 	"fmt"
 	"io"
 )
@@ -32,6 +33,14 @@ func PackValue(w io.Writer, barr []byte) error {
 	return nil
 }
 
+func PackStrArr(w io.Writer, strs []string) error {
+	barr, err := json.Marshal(strs)
+	if err != nil {
+		return err
+	}
+	return PackValue(w, barr)
+}
+
 func PackInt(w io.Writer, ival int) error {
 	viBuf := make([]byte, binary.MaxVarintLen64)
 	l := binary.PutUvarint(viBuf, uint64(ival))
@@ -55,6 +64,19 @@ func UnpackValue(r FullByteReader) ([]byte, error) {
 	return rtnBuf, nil
 }
 
+func UnpackStrArr(r FullByteReader) ([]string, error) {
+	barr, err := UnpackValue(r)
+	if err != nil {
+		return nil, err
+	}
+	var strs []string
+	err = json.Unmarshal(barr, &strs)
+	if err != nil {
+		return nil, err
+	}
+	return strs, nil
+}
+
 func UnpackInt(r io.ByteReader) (int, error) {
 	ival64, err := binary.ReadVarint(r)
 	if err != nil {
@@ -85,6 +107,17 @@ func (u *Unpacker) UnpackInt(name string) int {
 	return rtn
 }
 
+func (u *Unpacker) UnpackStrArr(name string) []string {
+	if u.Err != nil {
+		return nil
+	}
+	rtn, err := UnpackStrArr(u.R)
+	if err != nil {
+		u.Err = fmt.Errorf("cannot unpack %s: %v", name, err)
+	}
+	return rtn
+}
+
 func (u *Unpacker) Error() error {
 	return u.Err
 }

pkg/packet/shellstate.go

@@ -21,22 +21,32 @@ type ShellState struct {
 	Aliases   string `json:"aliases,omitempty"`
 	Funcs     string `json:"funcs,omitempty"`
 	Error     string `json:"error,omitempty"`
+	HashVal   string `json:"-"`
 }
 
 type ShellStateDiff struct {
 	Version     string   `json:"version"` // [type] [semver]
 	BaseHash    string   `json:"basehash"`
+	DiffHashArr []string `json:"diffhasharr,omitempty"`
 	Cwd         string   `json:"cwd,omitempty"`
 	VarsDiff    []byte   `json:"shellvarsdiff,omitempty"` // vardiff
 	AliasesDiff []byte   `json:"aliasesdiff,omitempty"`   // linediff
 	FuncsDiff   []byte   `json:"funcsdiff,omitempty"`     // linediff
 	Error       string   `json:"error,omitempty"`
+	HashVal     string   `json:"-"`
 }
 
 func (state ShellState) IsEmpty() bool {
 	return state.Version == "" && state.Cwd == "" && len(state.ShellVars) == 0 && state.Aliases == "" && state.Funcs == "" && state.Error == ""
 }
 
+// returns base64 hash of data
+func sha1Hash(data []byte) string {
+	hvalRaw := sha1.Sum(data)
+	hval := base64.StdEncoding.EncodeToString(hvalRaw[:])
+	return hval
+}
+
 // returns (SHA1, encoded-state)
 func (state ShellState) EncodeAndHash() (string, []byte) {
 	var buf bytes.Buffer
@@ -47,22 +57,24 @@ func (state ShellState) EncodeAndHash() (string, []byte) {
 	binpack.PackValue(&buf, []byte(state.Aliases))
 	binpack.PackValue(&buf, []byte(state.Funcs))
 	binpack.PackValue(&buf, []byte(state.Error))
-	hvalRaw := sha1.Sum(buf.Bytes())
+	return sha1Hash(buf.Bytes()), buf.Bytes()
-	hval := base64.StdEncoding.EncodeToString(hvalRaw[:])
-	return hval, buf.Bytes()
 }
 
 func (state ShellState) MarshalJSON() ([]byte, error) {
-	_, encodedState := state.EncodeAndHash()
+	_, encodedBytes := state.EncodeAndHash()
-	return json.Marshal(encodedState)
+	return json.Marshal(encodedBytes)
 }
 
-func (state *ShellState) UnmarshalJSON(jsonBytes []byte) error {
+// caches HashVal in struct
-	var barr []byte
+func (state *ShellState) GetHashVal(force bool) string {
-	err := json.Unmarshal(jsonBytes, &barr)
+	if state.HashVal == "" || force {
-	if err != nil {
+		state.HashVal, _ = state.EncodeAndHash()
-		return err
 	}
+	return state.HashVal
+}
+
+func (state *ShellState) DecodeShellState(barr []byte) error {
+	state.HashVal = sha1Hash(barr)
 	buf := bytes.NewBuffer(barr)
 	u := binpack.MakeUnpacker(buf)
 	version := u.UnpackInt("ShellState pack version")
@@ -78,25 +90,36 @@ func (state *ShellState) UnmarshalJSON(jsonBytes []byte) error {
 	return u.Error()
 }
 
-func (sdiff ShellStateDiff) MarshalJSON() ([]byte, error) {
+func (state *ShellState) UnmarshalJSON(jsonBytes []byte) error {
-	var buf bytes.Buffer
-	binpack.PackInt(&buf, ShellStateDiffPackVersion)
-	binpack.PackValue(&buf, []byte(sdiff.Version))
-	binpack.PackValue(&buf, []byte(sdiff.BaseHash))
-	binpack.PackValue(&buf, []byte(sdiff.Cwd))
-	binpack.PackValue(&buf, sdiff.VarsDiff)
-	binpack.PackValue(&buf, sdiff.AliasesDiff)
-	binpack.PackValue(&buf, sdiff.FuncsDiff)
-	binpack.PackValue(&buf, []byte(sdiff.Error))
-	return json.Marshal(buf.Bytes())
-}
-
-func (sdiff *ShellStateDiff) UnmarshalJSON(jsonBytes []byte) error {
 	var barr []byte
 	err := json.Unmarshal(jsonBytes, &barr)
 	if err != nil {
 		return err
 	}
+	return state.DecodeShellState(barr)
+}
+
+func (sdiff ShellStateDiff) EncodeAndHash() (string, []byte) {
+	var buf bytes.Buffer
+	binpack.PackInt(&buf, ShellStateDiffPackVersion)
+	binpack.PackValue(&buf, []byte(sdiff.Version))
+	binpack.PackValue(&buf, []byte(sdiff.BaseHash))
+	binpack.PackStrArr(&buf, sdiff.DiffHashArr)
+	binpack.PackValue(&buf, []byte(sdiff.Cwd))
+	binpack.PackValue(&buf, sdiff.VarsDiff)
+	binpack.PackValue(&buf, sdiff.AliasesDiff)
+	binpack.PackValue(&buf, sdiff.FuncsDiff)
+	binpack.PackValue(&buf, []byte(sdiff.Error))
+	return sha1Hash(buf.Bytes()), buf.Bytes()
+}
+
+func (sdiff ShellStateDiff) MarshalJSON() ([]byte, error) {
+	_, encodedBytes := sdiff.EncodeAndHash()
+	return json.Marshal(encodedBytes)
+}
+
+func (sdiff *ShellStateDiff) DecodeShellStateDiff(barr []byte) error {
+	sdiff.HashVal = sha1Hash(barr)
 	buf := bytes.NewBuffer(barr)
 	u := binpack.MakeUnpacker(buf)
 	version := u.UnpackInt("ShellState pack version")
@@ -105,6 +128,7 @@ func (sdiff *ShellStateDiff) UnmarshalJSON(jsonBytes []byte) error {
 	}
 	sdiff.Version = string(u.UnpackValue("ShellStateDiff.Version"))
 	sdiff.BaseHash = string(u.UnpackValue("ShellStateDiff.BaseHash"))
+	sdiff.DiffHashArr = u.UnpackStrArr("ShellStateDiff.DiffHashArr")
 	sdiff.Cwd = string(u.UnpackValue("ShellStateDiff.Cwd"))
 	sdiff.VarsDiff = u.UnpackValue("ShellStateDiff.VarsDiff")
 	sdiff.AliasesDiff = u.UnpackValue("ShellStateDiff.AliasesDiff")
@@ -113,6 +137,23 @@ func (sdiff *ShellStateDiff) UnmarshalJSON(jsonBytes []byte) error {
 	return u.Error()
 }
 
+func (sdiff *ShellStateDiff) UnmarshalJSON(jsonBytes []byte) error {
+	var barr []byte
+	err := json.Unmarshal(jsonBytes, &barr)
+	if err != nil {
+		return err
+	}
+	return sdiff.DecodeShellStateDiff(barr)
+}
+
+// caches HashVal in struct
+func (sdiff *ShellStateDiff) GetHashVal(force bool) string {
+	if sdiff.HashVal == "" || force {
+		sdiff.HashVal, _ = sdiff.EncodeAndHash()
+	}
+	return sdiff.HashVal
+}
+
 func (sdiff ShellStateDiff) Dump() {
 	fmt.Printf("ShellStateDiff:\n")
 	fmt.Printf("  version: %s\n", sdiff.Version)

pkg/shexec/parser.go

@@ -190,14 +190,13 @@ func ParseDeclLine(envLine string) *DeclareDeclType {
 	}
 }
 
+// returns name => full-line
 func parseDeclLineToKV(envLine string) (string, string) {
-	eqIdx := strings.Index(envLine, "=")
+	decl := ParseDeclLine(envLine)
-	if eqIdx == -1 {
+	if decl == nil {
 		return "", ""
 	}
-	namePart := envLine[0:eqIdx]
+	return decl.Name, envLine
-	valPart := envLine[eqIdx+1:]
-	return namePart, valPart
 }
 
 func shellStateVarsToMap(shellVars []byte) map[string]string {
@@ -216,6 +215,35 @@ func shellStateVarsToMap(shellVars []byte) map[string]string {
 	return rtn
 }
 
+func strMapToShellStateVars(varMap map[string]string) []byte {
+	var buf bytes.Buffer
+	orderedKeys := getOrderedKeysStrMap(varMap)
+	for _, key := range orderedKeys {
+		val := varMap[key]
+		buf.WriteString(val)
+		buf.WriteByte(0)
+	}
+	return buf.Bytes()
+}
+
+func getOrderedKeysStrMap(m map[string]string) []string {
+	keys := make([]string, 0, len(m))
+	for key, _ := range m {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+func getOrderedKeysDeclMap(m map[string]*DeclareDeclType) []string {
+	keys := make([]string, 0, len(m))
+	for key, _ := range m {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
 func DeclMapFromState(state *packet.ShellState) map[string]*DeclareDeclType {
 	if state == nil {
 		return nil
@@ -233,7 +261,9 @@ func DeclMapFromState(state *packet.ShellState) map[string]*DeclareDeclType {
 
 func SerializeDeclMap(declMap map[string]*DeclareDeclType) []byte {
 	var rtn bytes.Buffer
-	for _, decl := range declMap {
+	orderedKeys := getOrderedKeysDeclMap(declMap)
+	for _, key := range orderedKeys {
+		decl := declMap[key]
 		rtn.WriteString(decl.Serialize())
 	}
 	return rtn.Bytes()
@@ -269,6 +299,18 @@ func ShellVarMapFromState(state *packet.ShellState) map[string]string {
 	return rtn
 }
 
+func DumpVarMapFromState(state *packet.ShellState) {
+	fmt.Printf("DUMP-STATE-VARS:\n")
+	if state == nil {
+		fmt.Printf("  nil\n")
+		return
+	}
+	vars := bytes.Split(state.ShellVars, []byte{0})
+	for _, varLine := range vars {
+		fmt.Printf("  %s\n", varLine)
+	}
+}
+
 func VarDeclsFromState(state *packet.ShellState) []*DeclareDeclType {
 	if state == nil {
 		return nil
@@ -365,8 +407,8 @@ func parseDeclareOutput(state *packet.ShellState, declareBytes []byte) error {
 	if err != nil {
 		return err
 	}
-	var varsBuffer bytes.Buffer
 	var firstParseErr error
+	declMap := make(map[string]*DeclareDeclType)
 	for _, stmt := range file.Stmts {
 		decl, err := parseDeclareStmt(stmt, declareStr)
 		if err != nil {
@@ -375,10 +417,10 @@ func parseDeclareOutput(state *packet.ShellState, declareBytes []byte) error {
 			}
 		}
 		if decl != nil && !NoStoreVarNames[decl.Name] {
-			varsBuffer.WriteString(decl.Serialize())
+			declMap[decl.Name] = decl
 		}
 	}
-	state.ShellVars = varsBuffer.Bytes()
+	state.ShellVars = SerializeDeclMap(declMap) // this writes out the decls in a canonical order
 	if firstParseErr != nil {
 		state.Error = firstParseErr.Error()
 	}
@@ -533,3 +575,26 @@ func MakeShellStateDiff(oldState packet.ShellState, oldStateHash string, newStat
 	rtn.FuncsDiff = statediff.MakeLineDiff(oldState.Funcs, newState.Funcs)
 	return rtn, nil
 }
+
+func ApplyShellStateDiff(oldState packet.ShellState, diff packet.ShellStateDiff) (packet.ShellState, error) {
+	var rtnState packet.ShellState
+	var err error
+	rtnState.Version = oldState.Version
+	rtnState.Cwd = diff.Cwd
+	rtnState.Error = diff.Error
+	oldVars := shellStateVarsToMap(oldState.ShellVars)
+	newVars, err := statediff.ApplyMapDiff(oldVars, diff.VarsDiff)
+	if err != nil {
+		return rtnState, fmt.Errorf("applying mapdiff 'vars': %v", err)
+	}
+	rtnState.ShellVars = strMapToShellStateVars(newVars)
+	rtnState.Aliases, err = statediff.ApplyLineDiff(oldState.Aliases, diff.AliasesDiff)
+	if err != nil {
+		return rtnState, fmt.Errorf("applying diff 'aliases': %v", err)
+	}
+	rtnState.Funcs, err = statediff.ApplyLineDiff(oldState.Funcs, diff.FuncsDiff)
+	if err != nil {
+		return rtnState, fmt.Errorf("applying diff 'funcs': %v", err)
+	}
+	return rtnState, nil
+}

pkg/statediff/linediff.go

@@ -161,6 +161,9 @@ func makeLineDiff(oldData []string, newData []string) LineDiffType {
 }
 
 func MakeLineDiff(str1 string, str2 string) []byte {
+	if str1 == str2 {
+		return nil
+	}
 	str1Arr := strings.Split(str1, "\n")
 	str2Arr := strings.Split(str2, "\n")
 	diff := makeLineDiff(str1Arr, str2Arr)
@@ -168,6 +171,9 @@ func MakeLineDiff(str1 string, str2 string) []byte {
 }
 
 func ApplyLineDiff(str1 string, diffBytes []byte) (string, error) {
+	if len(diffBytes) == 0 {
+		return str1, nil
+	}
 	var diff LineDiffType
 	err := diff.Decode(diffBytes)
 	if err != nil {

pkg/statediff/mapdiff.go

@@ -111,10 +111,16 @@ func (diff *MapDiffType) Decode(diffBytes []byte) error {
 
 func MakeMapDiff(m1 map[string]string, m2 map[string]string) []byte {
 	diff := makeMapDiff(m1, m2)
+	if len(diff.ToAdd) == 0 && len(diff.ToRemove) == 0 {
+		return nil
+	}
 	return diff.Encode()
 }
 
 func ApplyMapDiff(oldMap map[string]string, diffBytes []byte) (map[string]string, error) {
+	if len(diffBytes) == 0 {
+		return oldMap, nil
+	}
 	var diff MapDiffType
 	err := diff.Decode(diffBytes)
 	if err != nil {