subparse backticks and dollar double quote

pkg/shparse/shparse.go

@@ -82,20 +82,18 @@ const (
 	WordTypeKey       = "key"  // if then else elif fi do done case esac while until for in { } ! (( [[
 	WordTypeSimpleVar = "svar" // simplevar $
 	WordTypeGroup     = "grp"  // contains other words e.g. "hello"foo'bar'$x
-	WordTypeArith     = "ath"
 
-	// each of these can also be used as an entry in quoteContext
+	WordTypeDQ       = "dq"   // "    (quote-context)
-	WordTypeDQ       = "dq"   // "
+	WordTypeDDQ      = "ddq"  // $"   (quote-context)
-	WordTypeSQ       = "sq"   // '
+	WordTypeVarBrace = "varb" // ${   (quote-context)
-	WordTypeBQ       = "bq"   // `
+	WordTypeDP       = "dp"   // $(   (quote-context)
-	WordTypeDSQ      = "dsq"  // $'
+	WordTypeBQ       = "bq"   // `    (quote-context)
-	WordTypeDDQ      = "ddq"  // $"
+
-	WordTypeVarBrace = "varb" // ${
+	WordTypeSQ  = "sq"  // '
-	WordTypeDP       = "dp"   // $(
+	WordTypeDSQ = "dsq" // $'
-	WordTypeDPP      = "dpp"  // $((
+	WordTypeDPP = "dpp" // $((   (internals not parsed)
-	WordTypeP        = "p"    // (
+	WordTypePP  = "pp"  // ((    (internals not parsed)
-	WordTypePP       = "pp"   // ((
+	WordTypeDB  = "db"  // $[    (internals not parsed)
-	WordTypeDB       = "db"   // $[
 )
 
 type quoteContext []string
@@ -268,12 +266,27 @@ func (c *parseContext) parseStrDQ() *wordType {
 	return w
 }
 
-func (c *parseContext) parseStrBQ() *wordType {
+func (c *parseContext) parseStrDDQ() *wordType {
-	if c.match('`') {
+	if !c.match2('$', '"') {
 		return nil
 	}
-	newOffset, complete := c.skipToChar(1, '`', true)
+	newContext := c.clone(c.Pos+2, WordTypeDDQ)
-	w := c.makeWord(WordTypeBQ, newOffset, complete)
+	subWords, eofExit := newContext.tokenizeDQ()
+	newOffset := newContext.Pos + 2
+	w := c.makeWord(WordTypeDDQ, newOffset, !eofExit)
+	w.Subs = subWords
+	return w
+}
+
+func (c *parseContext) parseStrBQ() *wordType {
+	if !c.match('`') {
+		return nil
+	}
+	newContext := c.clone(c.Pos+1, WordTypeBQ)
+	subWords, eofExit := newContext.tokenizeRaw()
+	newOffset := newContext.Pos + 1
+	w := c.makeWord(WordTypeBQ, newOffset, !eofExit)
+	w.Subs = subWords
 	return w
 }
 
@@ -286,15 +299,6 @@ func (c *parseContext) parseStrANSI() *wordType {
 	return w
 }
 
-func (c *parseContext) parseStrDDQ() *wordType {
-	if !c.match2('$', '"') {
-		return nil
-	}
-	newOffset, complete := c.skipToChar(2, '"', true)
-	w := c.makeWord(WordTypeDDQ, newOffset, complete)
-	return w
-}
-
 func (c *parseContext) parseArith(mustComplete bool) *wordType {
 	if !c.match2('(', '(') {
 		return nil
@@ -303,7 +307,7 @@ func (c *parseContext) parseArith(mustComplete bool) *wordType {
 	if mustComplete && !complete {
 		return nil
 	}
-	w := c.makeWord(WordTypeArith, newOffset, complete)
+	w := c.makeWord(WordTypePP, newOffset, complete)
 	return w
 }
 
@@ -358,20 +362,6 @@ func (c *parseContext) parseExpansion() *wordType {
 	return nil
 }
 
-func (c *parseContext) parseShellTest() *wordType {
-	if !c.match2('[', '[') {
-		return nil
-	}
-	return nil
-}
-
-func (c *parseContext) parseProcessSubstitution() *wordType {
-	if !c.match2('<', '(') && !c.match2('>', '(') {
-		return nil
-	}
-	return nil
-}
-
 // returns newOffset
 func (c *parseContext) parseSimpleVarName(offset int) int {
 	first := true

pkg/shparse/shparse_test.go

@@ -43,4 +43,6 @@ func Test1(t *testing.T) {
 	testParse(t, `echo $(ls $)`)
 	testParse(t, `echo ${x:-hello\}"}"} 2nd`)
 	testParse(t, `echo "$(ls "foo") more $x"`)
+	testParse(t, "echo `ls $x \"hello $x\" \\`ls\\`; ./foo`")
+	testParse(t, `echo $"hello $x $(ls)"`)
 }

pkg/shparse/tokenize.go

@@ -221,9 +221,12 @@ func (c *parseContext) tokenizeDQ() ([]*wordType, bool) {
 }
 
 // returns (words, eofexit)
+// backticks (WordTypeBQ) handle backslash in a special way, but that seems to mainly effect execution (not completion)
+//     de_backslash => removes initial backslash in \`, \\, and \$ before execution
 func (c *parseContext) tokenizeRaw() ([]*wordType, bool) {
 	state := &tokenizeOutputState{}
 	isExpSubShell := c.QC.cur() == WordTypeDP
+	isInBQ := c.QC.cur() == WordTypeBQ
 	parenLevel := 0
 	eofExit := false
 	for {
@@ -236,6 +239,10 @@ func (c *parseContext) tokenizeRaw() ([]*wordType, bool) {
 			c.Pos++
 			break
 		}
+		if isInBQ && ch == '`' {
+			c.Pos++
+			break
+		}
 		// fmt.Printf("ch %d %q\n", c.Pos, string([]rune{ch}))
 		foundOp, newOffset := c.parseOp(0)
 		if foundOp {
@@ -264,6 +271,9 @@ func (c *parseContext) tokenizeRaw() ([]*wordType, bool) {
 		if quoteWord == nil && ch == '"' {
 			quoteWord = c.parseStrDQ()
 		}
+		if quoteWord == nil && ch == '`' {
+			quoteWord = c.parseStrBQ()
+		}
 		isNextParen := isExpSubShell && c.at(1) == ')'
 		if quoteWord == nil && ch == '$' && !isNextParen {
 			quoteWord = c.parseStrANSI()