words -> str. test roundtrip

pkg/shparse/shparse_test.go

@@ -18,6 +18,11 @@ func testParse(t *testing.T, s string) {
 	fmt.Printf("%s\n", s)
 	dumpWords(words, "  ")
 	fmt.Printf("\n")
+
+	outStr := wordsToStr(words)
+	if outStr != s {
+		t.Errorf("tokenization output does not match input: %q => %q", s, outStr)
+	}
 }
 
 func Test1(t *testing.T) {

pkg/shparse/tokenize.go

@@ -1,6 +1,7 @@
 package shparse
 
 import (
+	"bytes"
 	"unicode"
 )
 
@@ -108,3 +109,14 @@ func Tokenize(cmd string) []*wordType {
 	state.finish(c)
 	return state.Rtn
 }
+
+func wordsToStr(words []*wordType) string {
+	var buf bytes.Buffer
+	for _, word := range words {
+		if len(word.Prefix) > 0 {
+			buf.WriteString(string(word.Prefix))
+		}
+		buf.WriteString(string(word.Raw))
+	}
+	return buf.String()
+}