waveterm/pkg/shparse/shparse.go

package shparse

import (
	"bytes"
	"fmt"
)

//
// cmds := cmd (sep cmd)*
// sep := ';' | '&' | '&&' | '||' | '|' | '\n'
// cmd := simple-cmd | compound-command redirect-list?
// compound-command := brace-group | subshell | for-clause | case-clause | if-clause | while-clause | until-clause
// brace-group := '{' cmds '}'
// subshell := '(' cmds ')'
// simple-command := cmd-prefix cmd-word (io-redirect)*
// cmd-prefix := (io-redirect | assignment)*
// cmd-suffix := (io-redirect | word)*
// cmd-name := word
// cmd-word := word
// io-redirect := (io-number? io-file) | (io-number? io-here)
// io-file := ('<' | '<&' | '>' | '>&' | '>>' | '>|' ) filename
// io-here := ('<<' | '<<-') here_end
// here-end := word
// if-clause := 'if' compound-list 'then' compound-list else-part 'fi'
// else-part :=   'elif' compound-list 'then' compound-list
//              | 'elif' compount-list 'then' compound-list else-part
//              | 'else' compound-list
// compound-list := linebreak term sep?
//
//
//
// A correctly-formed brace expansion must contain unquoted opening and closing braces, and at least one unquoted comma or a valid sequence expression
// Any incorrectly formed brace expansion is left unchanged.
//
// ambiguity between $((...)) and $((ls); ls)
// ambiguity between foo=([0]=hell) and foo=([abc)
// tokenization https://pubs.opengroup.org/onlinepubs/7908799/xcu/chap2.html#tag_001_003

// can-extend: WordTypeLit, WordTypeSimpleVar, WordTypeVarBrace, WordTypeDQ, WordTypeDDQ, WordTypeSQ, WordTypeDSQ
const (
	WordTypeRaw       = "raw"
	WordTypeLit       = "lit"  // (can-extend)
	WordTypeOp        = "op"   // single: & ; | ( ) < > \n  multi(2): && || ;; << >> <& >& <> >| ((  multi(3): <<-    ('((' requires special processing)
	WordTypeKey       = "key"  // if then else elif fi do done case esac while until for in { } ! (( [[
	WordTypeGroup     = "grp"  // contains other words e.g. "hello"foo'bar'$x
	WordTypeSimpleVar = "svar" // simplevar $ (can-extend)

	WordTypeDQ       = "dq"   // "    (quote-context) (can-extend)
	WordTypeDDQ      = "ddq"  // $"   (quote-context) (can-extend)
	WordTypeVarBrace = "varb" // ${   (quote-context) (can-extend)
	WordTypeDP       = "dp"   // $(   (quote-context)
	WordTypeBQ       = "bq"   // `    (quote-context)

	WordTypeSQ  = "sq"  // '     (can-extend)
	WordTypeDSQ = "dsq" // $'    (can-extend)
	WordTypeDPP = "dpp" // $((   (internals not parsed)
	WordTypePP  = "pp"  // ((    (internals not parsed)
	WordTypeDB  = "db"  // $[    (internals not parsed)
)

const (
	CmdTypeNone   = "none"   // holds control structures: '(' ')' 'for' 'while' etc.
	CmdTypeSimple = "simple" // holds real commands
)

type WordType struct {
	Type     string
	Offset   int
	QC       QuoteContext
	Raw      []rune
	Complete bool
	Prefix   []rune
	Subs     []*WordType
}

type CmdType struct {
	Type            string
	AssignmentWords []*WordType
	Words           []*WordType
}

type QuoteContext []string

var wordMetaMap map[string]wordMeta

// same order as https://www.gnu.org/software/bash/manual/html_node/Reserved-Words.html
var bashReservedWords = []string{
	"if", "then", "elif", "else", "fi", "time",
	"for", "in", "until", "while", "do", "done",
	"case", "esac", "coproc", "select", "function",
	"{", "}", "[[", "]]", "!",
}

// special reserved words: "for", "in", "case", "select", "function", "[[", and "]]"

var bashNoneRW = []string{
	"if", "then", "elif", "else", "fi", "time",
	"until", "while", "do", "done",
	"esac", "coproc",
	"{", "}", "!",
}

type wordMeta struct {
	Type         string
	EmptyWord    []rune
	SuffixLen    int
	CanExtend    bool
	QuoteContext bool
}

func makeWordMeta(wtype string, emptyWord string, suffixLen int, canExtend bool, quoteContext bool) {
	wordMetaMap[wtype] = wordMeta{wtype, []rune(emptyWord), suffixLen, canExtend, quoteContext}
}

func init() {
	wordMetaMap = make(map[string]wordMeta)
	makeWordMeta(WordTypeRaw, "", 0, false, false)
	makeWordMeta(WordTypeLit, "", 0, true, false)
	makeWordMeta(WordTypeOp, "", 0, false, false)
	makeWordMeta(WordTypeKey, "", 0, false, false)
	makeWordMeta(WordTypeGroup, "", 0, false, false)
	makeWordMeta(WordTypeSimpleVar, "$", 0, true, false)
	makeWordMeta(WordTypeVarBrace, "${}", 1, true, true)
	makeWordMeta(WordTypeDQ, `""`, 1, true, true)
	makeWordMeta(WordTypeDDQ, `$""`, 1, true, true)
	makeWordMeta(WordTypeDP, "$()", 1, false, false)
	makeWordMeta(WordTypeBQ, "``", 1, false, false)
	makeWordMeta(WordTypeSQ, "''", 1, true, false)
	makeWordMeta(WordTypeDSQ, "$''", 1, true, false)
	makeWordMeta(WordTypeDPP, "$(())", 2, false, false)
	makeWordMeta(WordTypePP, "(())", 2, false, false)
	makeWordMeta(WordTypeDB, "$[]", 1, false, false)
}

func MakeEmptyWord(wtype string, qc QuoteContext, offset int) *WordType {
	meta := wordMetaMap[wtype]
	if meta.Type == "" {
		meta = wordMetaMap[WordTypeRaw]
	}
	rtn := &WordType{Type: meta.Type, QC: qc, Offset: offset, Complete: true}
	if len(meta.EmptyWord) > 0 {
		rtn.Raw = append([]rune(nil), meta.EmptyWord...)
	}
	return rtn
}

func (qc QuoteContext) push(q string) QuoteContext {
	rtn := make([]string, 0, len(qc)+1)
	rtn = append(rtn, qc...)
	rtn = append(rtn, q)
	return rtn
}

func (qc QuoteContext) cur() string {
	if len(qc) == 0 {
		return ""
	}
	return qc[len(qc)-1]
}

func makeRepeatStr(ch byte, slen int) string {
	if slen == 0 {
		return ""
	}
	rtn := make([]byte, slen)
	for i := 0; i < slen; i++ {
		rtn[i] = ch
	}
	return string(rtn)
}

func (w *WordType) String() string {
	notCompleteFlag := " "
	if !w.Complete {
		notCompleteFlag = "*"
	}
	return fmt.Sprintf("%4s[%3d]%s %s%q", w.Type, w.Offset, notCompleteFlag, makeRepeatStr('_', len(w.Prefix)), string(w.FullRawString()))
}

func dumpWords(words []*WordType, indentStr string) {
	for _, word := range words {
		fmt.Printf("%s%s\n", indentStr, word.String())
		if len(word.Subs) > 0 {
			dumpWords(word.Subs, indentStr+"  ")
		}
	}
}

func dumpCommands(cmds []*CmdType, indentStr string) {
	for _, cmd := range cmds {
		fmt.Printf("%sCMD: %s [%d]\n", indentStr, cmd.Type, len(cmd.Words))
		dumpWords(cmd.Words, indentStr+"  ")
	}
}

func (w *WordType) FullRawString() []rune {
	if w.Type == WordTypeGroup {
		var rtn []rune
		for _, sw := range w.Subs {
			rtn = append(rtn, sw.FullRawString()...)
		}
		return rtn
	}
	return w.Raw
}

func wordsToStr(words []*WordType) string {
	var buf bytes.Buffer
	for _, word := range words {
		if len(word.Prefix) > 0 {
			buf.WriteString(string(word.Prefix))
		}
		buf.WriteString(string(word.FullRawString()))
	}
	return buf.String()
}

// recognizes reserved words in first position
func convertToAnyReservedWord(w *WordType) bool {
	if w == nil || w.Type != WordTypeLit {
		return false
	}
	rawVal := string(w.Raw)
	for _, rw := range bashReservedWords {
		if rawVal == rw {
			w.Type = WordTypeKey
			return true
		}
	}
	return false
}

// recognizes the specific reserved-word given only ('in' and 'do' in 'for', 'case', and 'select' commands)
func convertToReservedWord(w *WordType, reservedWord string) {
	if w == nil || w.Type != WordTypeLit {
		return
	}
	if string(w.Raw) == reservedWord {
		w.Type = WordTypeKey
	}
}

func isNoneReservedWord(w *WordType) bool {
	if w.Type != WordTypeKey {
		return false
	}
	rawVal := string(w.Raw)
	for _, rw := range bashNoneRW {
		if rawVal == rw {
			return true
		}
	}
	return false
}

type parseCmdState struct {
	Input    []*WordType
	InputPos int

	Rtn []*CmdType
	Cur *CmdType
}

func (state *parseCmdState) isEof() bool {
	return state.InputPos >= len(state.Input)
}

func (state *parseCmdState) curWord() *WordType {
	if state.isEof() {
		return nil
	}
	return state.Input[state.InputPos]
}

func (state *parseCmdState) lastCmd() *CmdType {
	if len(state.Rtn) == 0 {
		return nil
	}
	return state.Rtn[len(state.Rtn)-1]
}

func (state *parseCmdState) makeNoneCmd() {
	lastCmd := state.lastCmd()
	if lastCmd == nil || lastCmd.Type != CmdTypeNone {
		lastCmd = &CmdType{Type: CmdTypeNone}
		state.Rtn = append(state.Rtn, lastCmd)
	}
	lastCmd.Words = append(lastCmd.Words, state.curWord())
	state.Cur = nil
	state.InputPos++
}

func (state *parseCmdState) handleKeyword(word *WordType) bool {
	if word.Type != WordTypeKey {
		return false
	}
	if isNoneReservedWord(word) {
		state.makeNoneCmd()
		return true
	}
	rw := string(word.Raw)
	if rw == "[[" {
		// just ignore everything between [[ and ]]
		for !state.isEof() {
			curWord := state.curWord()
			if curWord.Type == WordTypeLit && string(curWord.Raw) == "]]" {
				convertToReservedWord(curWord, "]]")
				state.makeNoneCmd()
				break
			}
			state.makeNoneCmd()
		}
		return true
	}
	if rw == "case" {
		// ignore everything between "case" and "esac"
		for !state.isEof() {
			curWord := state.curWord()
			if curWord.Type == WordTypeKey && string(curWord.Raw) == "esac" {
				state.makeNoneCmd()
				break
			}
			state.makeNoneCmd()
		}
		return true
	}
	if rw == "for" || rw == "select" {
		// ignore until a "do"
		for !state.isEof() {
			curWord := state.curWord()
			if curWord.Type == WordTypeKey && string(curWord.Raw) == "do" {
				state.makeNoneCmd()
				break
			}
			state.makeNoneCmd()
		}
		return true
	}
	if rw == "in" {
		// the "for" and "case" clauses should skip "in".  so encountering an "in" here is a syntax error.
		// just treat it as a none and allow a new command after.
		state.makeNoneCmd()
		return true
	}
	if rw == "function" {
		// ignore until '{'
		for !state.isEof() {
			curWord := state.curWord()
			if curWord.Type == WordTypeKey && string(curWord.Raw) == "{" {
				state.makeNoneCmd()
				break
			}
			state.makeNoneCmd()
		}
		return true
	}
	state.makeNoneCmd()
	return true
}

func isCmdSeparatorOp(word *WordType) bool {
	if word.Type != WordTypeOp {
		return false
	}
	opVal := string(word.Raw)
	return opVal == ";" || opVal == "\n" || opVal == "&" || opVal == "|" || opVal == "|&" || opVal == "&&" || opVal == "||" || opVal == "(" || opVal == ")"
}

func (state *parseCmdState) handleOp(word *WordType) bool {
	opVal := string(word.Raw)
	// sequential separators
	if opVal == ";" || opVal == "\n" {
		state.makeNoneCmd()
		return true
	}
	// separator
	if opVal == "&" {
		state.makeNoneCmd()
		return true
	}
	// pipelines
	if opVal == "|" || opVal == "|&" {
		state.makeNoneCmd()
		return true
	}
	// lists
	if opVal == "&&" || opVal == "||" {
		state.makeNoneCmd()
		return true
	}
	// subshell
	if opVal == "(" || opVal == ")" {
		state.makeNoneCmd()
		return true
	}
	return false
}

func wordSliceBoundedIdx(words []*WordType, idx int) *WordType {
	if idx >= len(words) {
		return nil
	}
	return words[idx]
}

// note that a newline "op" can appear in the third position of "for" or "case".  the "in" keyword is still converted because of wordNum == 0
func identifyReservedWords(words []*WordType) {
	wordNum := 0
	lastReserved := false
	for idx, word := range words {
		if wordNum == 0 || lastReserved {
			convertToAnyReservedWord(word)
		}
		if word.Type == WordTypeKey {
			rwVal := string(word.Raw)
			switch rwVal {
			case "for":
				lastReserved = false
				third := wordSliceBoundedIdx(words, idx+2)
				convertToReservedWord(third, "in")
				convertToReservedWord(third, "do")

			case "case":
				lastReserved = false
				third := wordSliceBoundedIdx(words, idx+2)
				convertToReservedWord(third, "in")

			case "in":
				lastReserved = false

			default:
				lastReserved = true
			}
			continue
		}
		lastReserved = false
		if isCmdSeparatorOp(word) {
			wordNum = 0
			continue
		}
		wordNum++
	}
}

func ParseCommands(words []*WordType) []*CmdType {
	identifyReservedWords(words)
	state := parseCmdState{Input: words}
	for {
		if state.isEof() {
			break
		}
		word := state.curWord()
		if word.Type == WordTypeKey {
			done := state.handleKeyword(word)
			if done {
				continue
			}
		}
		if word.Type == WordTypeOp {
			done := state.handleOp(word)
			if done {
				continue
			}
		}
		if state.Cur == nil {
			state.Cur = &CmdType{Type: CmdTypeSimple}
			state.Rtn = append(state.Rtn, state.Cur)
		}
		state.Cur.Words = append(state.Cur.Words, word)
		state.InputPos++
	}
	return state.Rtn
}