checkpoint, extend working with all the crazy quote balancing for subs

pkg/shparse/extend.go

@@ -99,8 +99,74 @@ func (ec *extendContext) ensureCurWord() {
 }
 
 // grp, dq, ddq
-func extendWithSubs(buf *bytes.Buffer, word *WordType, wordPos int, extStr string) {
-
+func extendWithSubs(word *WordType, wordPos int, extStr string, complete bool) utilfn.StrWithPos {
+	wmeta := wordMetaMap[word.Type]
+	if word.Type == WordTypeGroup {
+		atEnd := (wordPos == len(word.Raw))
+		subWord := findCompletionWordAtPos(word.Subs, wordPos, true)
+		if subWord == nil {
+			strPos := Extend(MakeEmptyWord(WordTypeLit, word.QC, 0, true), 0, extStr, atEnd)
+			strPos = strPos.Prepend(string(word.Raw[0:wordPos]))
+			strPos = strPos.Append(string(word.Raw[wordPos:]))
+			return strPos
+		} else {
+			subComplete := complete && atEnd
+			strPos := Extend(subWord, wordPos-subWord.Offset, extStr, subComplete)
+			strPos = strPos.Prepend(string(word.Raw[0:subWord.Offset]))
+			strPos = strPos.Append(string(word.Raw[subWord.Offset+len(subWord.Raw):]))
+			return strPos
+		}
+	} else if word.Type == WordTypeDQ || word.Type == WordTypeDDQ {
+		if wordPos < word.contentStartPos() {
+			wordPos = word.contentStartPos()
+		}
+		atEnd := (wordPos >= len(word.Raw)-wmeta.SuffixLen)
+		subWord := findCompletionWordAtPos(word.Subs, wordPos-wmeta.PrefixLen, true)
+		quoteBalance := !atEnd
+		if subWord == nil {
+			realOffset := wordPos
+			strPos, wordOpen := extendInternal(MakeEmptyWord(WordTypeLit, word.QC.push(WordTypeDQ), 0, true), 0, extStr, false, quoteBalance)
+			strPos = strPos.Prepend(string(word.Raw[0:realOffset]))
+			var requiredSuffix string
+			if wordOpen {
+				requiredSuffix = wmeta.getSuffix()
+			}
+			if atEnd {
+				if complete {
+					return utilfn.StrWithPos{Str: strPos.Str + requiredSuffix + " ", Pos: strPos.Pos + len(requiredSuffix) + 1}
+				} else {
+					if word.Complete && requiredSuffix != "" {
+						return strPos.Append(requiredSuffix)
+					}
+					return strPos
+				}
+			}
+			strPos = strPos.Append(string(word.Raw[wordPos:]))
+			return strPos
+		} else {
+			realOffset := subWord.Offset + wmeta.PrefixLen
+			strPos, wordOpen := extendInternal(subWord, wordPos-realOffset, extStr, false, quoteBalance)
+			strPos = strPos.Prepend(string(word.Raw[0:realOffset]))
+			var requiredSuffix string
+			if wordOpen {
+				requiredSuffix = wmeta.getSuffix()
+			}
+			if atEnd {
+				if complete {
+					return utilfn.StrWithPos{Str: strPos.Str + requiredSuffix + " ", Pos: strPos.Pos + len(requiredSuffix) + 1}
+				} else {
+					if word.Complete && requiredSuffix != "" {
+						return strPos.Append(requiredSuffix)
+					}
+					return strPos
+				}
+			}
+			strPos = strPos.Append(string(word.Raw[realOffset+len(subWord.Raw):]))
+			return strPos
+		}
+	} else {
+		return utilfn.StrWithPos{Str: string(word.Raw), Pos: wordPos}
+	}
 }
 
 // lit, svar, varb, sq, dsq
@@ -127,6 +193,18 @@ func extendLeafCh(buf *bytes.Buffer, wordOpen *bool, wtype string, qc QuoteConte
 	}
 }
 
+func getWordOpenStr(wtype string, qc QuoteContext) string {
+	if wtype == WordTypeLit {
+		if qc.cur() == WordTypeDQ {
+			return "\""
+		} else {
+			return ""
+		}
+	}
+	wmeta := wordMetaMap[wtype]
+	return wmeta.getPrefix()
+}
+
 // lit, svar, varb sq, dsq
 func extendLeaf(buf *bytes.Buffer, wordOpen *bool, word *WordType, wordPos int, extStr string) {
 	for _, ch := range extStr {
@@ -135,45 +213,61 @@ func extendLeaf(buf *bytes.Buffer, wordOpen *bool, word *WordType, wordPos int,
 }
 
 // lit, grp, svar, dq, ddq, varb, sq, dsq
-func Extend(word *WordType, wordPos int, extStr string, complete bool) utilfn.StrWithPos {
+// returns (strwithpos, dq-closed)
+func extendInternal(word *WordType, wordPos int, extStr string, complete bool, requiresQuoteBalance bool) (utilfn.StrWithPos, bool) {
 	if extStr == "" {
-		return utilfn.StrWithPos{Str: string(word.Raw), Pos: wordPos}
+		return utilfn.StrWithPos{Str: string(word.Raw), Pos: wordPos}, true
+	}
+	if word.canHaveSubs() {
+		return extendWithSubs(word, wordPos, extStr, complete), true
 	}
 	var buf bytes.Buffer
 	isEOW := wordPos >= word.contentEndPos()
 	if isEOW {
 		wordPos = word.contentEndPos()
 	}
-	if wordPos > 0 && wordPos < word.contentStartPos() {
+	if wordPos < word.contentStartPos() {
 		wordPos = word.contentStartPos()
 	}
-	wordOpen := false
-	if wordPos >= word.contentStartPos() {
-		wordOpen = true
+	if wordPos > 0 {
+		buf.WriteString(string(word.Raw[0:word.contentStartPos()])) // write the prefix
 	}
-	buf.WriteString(string(word.Raw[0:wordPos])) // write the prefix
-	if word.canHaveSubs() {
-		extendWithSubs(&buf, word, wordPos, extStr)
-	} else {
-		extendLeaf(&buf, &wordOpen, word, wordPos, extStr)
+	if wordPos > word.contentStartPos() {
+		buf.WriteString(string(word.Raw[word.contentStartPos():wordPos]))
 	}
+	wordOpen := true
+	extendLeaf(&buf, &wordOpen, word, wordPos, extStr)
 	if isEOW {
 		// end-of-word, write the suffix (and optional ' ').  return the end of the string
 		wmeta := wordMetaMap[word.Type]
+		rtnPos := utf8.RuneCount(buf.Bytes())
 		buf.WriteString(wmeta.getSuffix())
-		var rtnPos int
+		if !wordOpen && requiresQuoteBalance {
+			buf.WriteString(getWordOpenStr(word.Type, word.QC))
+			wordOpen = true
+		}
 		if complete {
 			buf.WriteRune(' ')
-			rtnPos = utf8.RuneCount(buf.Bytes())
+			return utilfn.StrWithPos{Str: buf.String(), Pos: utf8.RuneCount(buf.Bytes())}, wordOpen
 		} else {
-			rtnPos = utf8.RuneCount(buf.Bytes()) - wmeta.SuffixLen
+			return utilfn.StrWithPos{Str: buf.String(), Pos: rtnPos}, wordOpen
 		}
-		return utilfn.StrWithPos{Str: buf.String(), Pos: rtnPos}
 	}
 	// completion in the middle of a word (no ' ')
 	rtnPos := utf8.RuneCount(buf.Bytes())
+	if !wordOpen {
+		// always required since there is a suffix
+		buf.WriteString(getWordOpenStr(word.Type, word.QC))
+		wordOpen = true
+	}
 	buf.WriteString(string(word.Raw[wordPos:])) // write the suffix
-	return utilfn.StrWithPos{Str: buf.String(), Pos: rtnPos}
+	return utilfn.StrWithPos{Str: buf.String(), Pos: rtnPos}, wordOpen
+}
+
+// lit, grp, svar, dq, ddq, varb, sq, dsq
+func Extend(word *WordType, wordPos int, extStr string, complete bool) utilfn.StrWithPos {
+	rtn, _ := extendInternal(word, wordPos, extStr, complete, false)
+	return rtn
 }
 
 func (ec *extendContext) extend(ch rune) {
@@ -204,7 +298,11 @@ func getSpecialEscape(ch rune) string {
 	return specialEsc[byte(ch)]
 }
 
-func writeSpecial(buf *bytes.Buffer, ch rune) {
+func writeSpecial(buf *bytes.Buffer, ch rune, wrap bool) {
+	if wrap {
+		buf.WriteRune('$')
+		buf.WriteRune('\'')
+	}
 	sesc := getSpecialEscape(ch)
 	if sesc != "" {
 		buf.WriteString(sesc)
@@ -212,6 +310,9 @@ func writeSpecial(buf *bytes.Buffer, ch rune) {
 		utf8Lit := getUtf8Literal(ch)
 		buf.WriteString(utf8Lit)
 	}
+	if wrap {
+		buf.WriteRune('\'')
+	}
 }
 
 func extendLit(buf *bytes.Buffer, ch rune) {
@@ -219,7 +320,7 @@ func extendLit(buf *bytes.Buffer, ch rune) {
 		return
 	}
 	if ch > unicode.MaxASCII || !unicode.IsPrint(ch) {
-		writeSpecial(buf, ch)
+		writeSpecial(buf, ch, true)
 		return
 	}
 	var bch = byte(ch)
@@ -236,19 +337,15 @@ func extendDSQ(buf *bytes.Buffer, wordOpen *bool, ch rune) {
 	if ch == 0 {
 		return
 	}
-	if ch > unicode.MaxASCII || !unicode.IsPrint(ch) {
-		if *wordOpen {
-			buf.WriteRune('\'')
-			*wordOpen = false
-		}
-		writeSpecial(buf, ch)
-		return
-	}
-	if *wordOpen {
+	if !*wordOpen {
 		buf.WriteRune('$')
 		buf.WriteRune('\'')
 		*wordOpen = true
 	}
+	if ch > unicode.MaxASCII || !unicode.IsPrint(ch) {
+		writeSpecial(buf, ch, false)
+		return
+	}
 	if ch == '\'' {
 		buf.WriteRune('\\')
 		buf.WriteRune(ch)
@@ -276,7 +373,7 @@ func extendSQ(buf *bytes.Buffer, wordOpen *bool, ch rune) {
 			buf.WriteRune('\'')
 			*wordOpen = false
 		}
-		writeSpecial(buf, ch)
+		writeSpecial(buf, ch, true)
 		return
 	}
 	if !*wordOpen {
@@ -296,7 +393,7 @@ func extendDQLit(buf *bytes.Buffer, wordOpen *bool, ch rune) {
 			buf.WriteRune('"')
 			*wordOpen = false
 		}
-		writeSpecial(buf, ch)
+		writeSpecial(buf, ch, true)
 		return
 	}
 	if !*wordOpen {

pkg/shparse/shparse.go

@@ -48,7 +48,7 @@ const (
 	WordTypeSimpleVar = "svar" // simplevar $ (can-extend)
 
 	WordTypeDQ       = "dq"   // "    (quote-context) (can-extend) (has-subs)
-	WordTypeDDQ      = "ddq"  // $"   (quote-context) (can-extend) (has-subs)
+	WordTypeDDQ      = "ddq"  // $"   (can-extend) (has-subs) (for quotecontext, uses WordTypeDQ)
 	WordTypeVarBrace = "varb" // ${   (quote-context) (can-extend) (internals not parsed)
 	WordTypeDP       = "dp"   // $(   (quote-context) (has-subs)
 	WordTypeBQ       = "bq"   // `    (quote-context) (has-subs)

pkg/shparse/shparse_test.go

@@ -66,9 +66,9 @@ func testExtend(t *testing.T, startStr string, extendStr string, complete bool,
 	}
 	outSP := Extend(word, startSP.Pos-word.Offset, extendStr, complete)
 	expSP := utilfn.ParseToSP(expStr)
-	fmt.Printf("extend: [%s] + [%s] => [%s]\n", startStr, extendStr, outSP)
+	fmt.Printf("extend: [%s] + %q => [%s]\n", startStr, extendStr, outSP)
 	if outSP != expSP {
-		t.Errorf("extension does not match: [%s] + [%s] => [%s] expected [%s]\n", startStr, extendStr, outSP, expSP)
+		t.Errorf("extension does not match: [%s] + %q => [%s] expected [%s]\n", startStr, extendStr, outSP, expSP)
 	}
 }
 
@@ -84,6 +84,19 @@ func Test2(t *testing.T) {
 	testExtend(t, `$f[*]`, "oo", false, "$foo[*]")
 	testExtend(t, `${f}[*]`, "oo", false, "${foo[*]}")
 	testExtend(t, `${f[*]}`, "oo", true, "${foo} [*]")
+	testExtend(t, `[*]`, "more stuff", false, `more\ stuff[*]`)
+	testExtend(t, `[*]`, "hello\amike", false, `hello$'\a'mike[*]`)
+	testExtend(t, `$'he[*]'`, "\x01\x02\x0a", true, `$'he\x01\x02\n' [*]`)
+	testExtend(t, `${x}\ [*]ll$y`, "e", false, `${x}\ e[*]ll$y`)
+	testExtend(t, `"he[*]"`, "$$o", true, `"he\$\$o" [*]`)
+	testExtend(t, `"h[*]llo"`, "e", false, `"he[*]llo"`)
+	testExtend(t, `"h[*]llo"`, "e", true, `"he[*]llo"`)
+	testExtend(t, `"[*]${h}llo"`, "e\x01", true, `"e"$'\x01'[*]"${h}llo"`)
+	testExtend(t, `"${h}llo[*]"`, "e\x01", true, `"${h}lloe"$'\x01' [*]`)
+	testExtend(t, `"${h}llo[*]"`, "e\x01", false, `"${h}lloe"$'\x01'[*]`)
+	testExtend(t, `"${h}ll[*]o"`, "e\x01", false, `"${h}lle"$'\x01'[*]"o"`)
+	testExtend(t, `"ab[*]c${x}def"`, "\x01", false, `"ab"$'\x01'[*]"c${x}def"`)
+	testExtend(t, `'ab[*]ef'`, "\x01", false, `'ab'$'\x01'[*]'ef'`)
 
 	// testExtend(t, `'he'`, "llo", `'hello'`)
 	// testExtend(t, `'he'`, "'", `'he'\'''`)

pkg/shparse/tokenize.go

@@ -472,7 +472,7 @@ func (c *parseContext) parseStrDDQ() *WordType {
 	if !c.match2('$', '"') {
 		return nil
 	}
-	newContext := c.clone(c.Pos+2, WordTypeDDQ)
+	newContext := c.clone(c.Pos+2, WordTypeDQ) // use WordTypeDQ (not DDQ)
 	subWords, eofExit := newContext.tokenizeDQ()
 	newOffset := newContext.Pos + 2
 	w := c.makeWord(WordTypeDDQ, newOffset, !eofExit)

pkg/utilfn/utilfn.go

@@ -157,3 +157,11 @@ func strWithCursor(str string, pos int) string {
 	}
 	return string(rtn)
 }
+
+func (sp StrWithPos) Prepend(str string) StrWithPos {
+	return StrWithPos{Str: str + sp.Str, Pos: utf8.RuneCountInString(str) + sp.Pos}
+}
+
+func (sp StrWithPos) Append(str string) StrWithPos {
+	return StrWithPos{Str: sp.Str + str, Pos: sp.Pos}
+}