From e6d7f749646026f6f9303d56563df48f5e4bba39 Mon Sep 17 00:00:00 2001 From: Mike Sawka Date: Fri, 15 Nov 2024 14:02:33 -0800 Subject: [PATCH] working on a buffered log viewer class (#1301) --- pkg/util/logview/logview.go | 217 +++++++++++++++++++++++++++++++++++ pkg/util/logview/multibuf.go | 131 +++++++++++++++++++++ 2 files changed, 348 insertions(+) create mode 100644 pkg/util/logview/logview.go create mode 100644 pkg/util/logview/multibuf.go diff --git a/pkg/util/logview/logview.go b/pkg/util/logview/logview.go new file mode 100644 index 000000000..f69dc4ae7 --- /dev/null +++ b/pkg/util/logview/logview.go @@ -0,0 +1,217 @@ +// Copyright 2024, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package logview + +import ( + "fmt" + "io" + "os" + "regexp" +) + +const BufSize = 256 * 1024 +const MaxLineSize = 1024 + +type LinePtr struct { + Offset int64 + RealLineNum int64 + LineNum int64 +} + +type LogView struct { + File *os.File + MultiBuf *MultiBufferByteGetter + MatchRe *regexp.Regexp +} + +func MakeLogView(file *os.File) *LogView { + return &LogView{ + File: file, + MultiBuf: MakeMultiBufferByteGetter(file, BufSize), + } +} + +func (lv *LogView) Close() { + lv.File.Close() +} + +func (lv *LogView) ReadLineData(linePtr *LinePtr) ([]byte, error) { + return lv.readLineAt(linePtr.Offset) +} + +func (lv *LogView) readLineAt(offset int64) ([]byte, error) { + var rtn []byte + for { + if len(rtn) > MaxLineSize { + break + } + b, err := lv.MultiBuf.GetByte(offset) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if b == '\n' { + break + } + rtn = append(rtn, b) + offset++ + } + return rtn, nil +} + +func (lv *LogView) FirstLinePtr() (*LinePtr, error) { + linePtr := &LinePtr{Offset: 0, RealLineNum: 1, LineNum: 1} + if lv.isLineMatch(0) { + return linePtr, nil + } + return lv.NextLinePtr(linePtr) +} + +func (lv *LogView) isLineMatch(offset int64) bool { + if lv.MatchRe == nil { + return true + } + lineData, err := lv.readLineAt(offset) + if err != nil { + return false + } + return lv.MatchRe.Match(lineData) +} + +func (lv *LogView) NextLinePtr(linePtr *LinePtr) (*LinePtr, error) { + if linePtr == nil { + return nil, fmt.Errorf("linePtr is nil") + } + numLines := int64(0) + offset := linePtr.Offset + for { + var err error + nextOffset, err := lv.MultiBuf.NextLine(offset) + if err == io.EOF { + return nil, nil + } + if err != nil { + return nil, err + } + numLines++ + if lv.isLineMatch(nextOffset) { + return &LinePtr{Offset: nextOffset, RealLineNum: linePtr.RealLineNum + numLines, LineNum: linePtr.LineNum + 1}, nil + } + offset = nextOffset + } +} + +func (lv *LogView) PrevLinePtr(linePtr *LinePtr) (*LinePtr, error) { + if linePtr == nil { + return nil, fmt.Errorf("linePtr is nil") + } + numLines := int64(0) + offset := linePtr.Offset + for { + var err error + prevOffset, err := lv.MultiBuf.PrevLine(offset) + if err == ErrBOF { + return nil, nil + } + if err != nil { + return nil, err + } + numLines++ + if lv.isLineMatch(prevOffset) { + return &LinePtr{Offset: prevOffset, RealLineNum: linePtr.RealLineNum - numLines, LineNum: linePtr.LineNum - 1}, nil + } + offset = prevOffset + } +} + +func (lv *LogView) Move(linePtr *LinePtr, offset int) (int, *LinePtr, error) { + var n int + if offset > 0 { + for { + nextLinePtr, err := lv.NextLinePtr(linePtr) + if err == io.EOF { + break + } + if err != nil { + return 0, nil, err + } + linePtr = nextLinePtr + n++ + if n == offset { + break + } + } + return n, linePtr, nil + } + if offset < 0 { + for { + prevLinePtr, err := lv.PrevLinePtr(linePtr) + if err == ErrBOF { + break + } + if err != nil { + return 0, nil, err + } + linePtr = prevLinePtr + n-- + if n == offset { + break + } + } + return n, linePtr, nil + } + return 0, linePtr, nil +} + +func (lv *LogView) LastLinePtr(linePtr *LinePtr) (*LinePtr, error) { + if linePtr == nil { + var err error + linePtr, err = lv.FirstLinePtr() + if err != nil { + return nil, err + } + } + if linePtr == nil { + return nil, nil + } + for { + nextLinePtr, err := lv.NextLinePtr(linePtr) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if nextLinePtr == nil { + break + } + linePtr = nextLinePtr + } + return linePtr, nil +} + +func (lv *LogView) ReadWindow(linePtr *LinePtr, winSize int) ([][]byte, error) { + if linePtr == nil { + return nil, nil + } + var rtn [][]byte + for len(rtn) < winSize { + lineData, err := lv.readLineAt(linePtr.Offset) + if err != nil { + return nil, err + } + rtn = append(rtn, lineData) + nextLinePtr, err := lv.NextLinePtr(linePtr) + if err != nil { + return nil, err + } + if nextLinePtr == nil { + break + } + linePtr = nextLinePtr + } + return rtn, nil +} diff --git a/pkg/util/logview/multibuf.go b/pkg/util/logview/multibuf.go new file mode 100644 index 000000000..78fa12678 --- /dev/null +++ b/pkg/util/logview/multibuf.go @@ -0,0 +1,131 @@ +// Copyright 2024, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package logview + +import ( + "errors" + "io" + "os" +) + +type MultiBufferByteGetter struct { + File *os.File + Offset int64 + EOF bool + Buffers [][]byte + BufSize int64 +} + +var ErrBOF = errors.New("beginning of file") + +func MakeMultiBufferByteGetter(file *os.File, bufSize int64) *MultiBufferByteGetter { + return &MultiBufferByteGetter{ + File: file, + Offset: 0, + EOF: false, + Buffers: [][]byte{}, + BufSize: bufSize, + } +} + +func (mb *MultiBufferByteGetter) readFromBuffer(offset int64) (byte, bool) { + if offset < mb.Offset || offset >= mb.Offset+int64(mb.bufSize()) { + return 0, false + } + bufIdx := int((offset - mb.Offset) / mb.BufSize) + bufOffset := (offset - mb.Offset) % mb.BufSize + return mb.Buffers[bufIdx][bufOffset], true +} + +func (mb *MultiBufferByteGetter) bufSize() int { + return len(mb.Buffers) * int(mb.BufSize) +} + +func (mb *MultiBufferByteGetter) rebuffer(newOffset int64) error { + partNum := int(newOffset / mb.BufSize) + partOffset := int64(partNum) * mb.BufSize + newBuf := make([]byte, mb.BufSize) + n, err := mb.File.ReadAt(newBuf, partOffset) + var isEOF bool + if err == io.EOF { + newBuf = newBuf[:n] + isEOF = true + } + if err != nil { + return err + } + var newBuffers [][]byte + if len(mb.Buffers) > 0 { + firstBufPartNum := int(mb.Offset / mb.BufSize) + lastBufPartNum := int((mb.Offset + int64(mb.bufSize())) / mb.BufSize) + if firstBufPartNum == partNum+1 { + newBuffers = [][]byte{newBuf, mb.Buffers[0]} + } else if lastBufPartNum == partNum-1 { + newBuffers = [][]byte{mb.Buffers[0], newBuf} + } else { + newBuffers = [][]byte{newBuf} + } + } else { + newBuffers = [][]byte{newBuf} + } + mb.Buffers = newBuffers + mb.Offset = partOffset + mb.EOF = isEOF + return nil +} + +func (mb *MultiBufferByteGetter) GetByte(offset int64) (byte, error) { + b, ok := mb.readFromBuffer(offset) + if ok { + return b, nil + } + if mb.EOF && offset >= mb.Offset+int64(mb.bufSize()) { + return 0, io.EOF + } + err := mb.rebuffer(offset) + if err != nil { + return 0, err + } + b, _ = mb.readFromBuffer(offset) + return b, nil +} + +func (mb *MultiBufferByteGetter) NextLine(offset int64) (int64, error) { + for { + b, err := mb.GetByte(offset) + if err != nil { + return 0, err + } + if b == '\n' { + break + } + offset++ + } + _, lastErr := mb.GetByte(offset + 1) + if lastErr == io.EOF { + return 0, io.EOF + } + return offset + 1, nil +} + +func (mb *MultiBufferByteGetter) PrevLine(offset int64) (int64, error) { + if offset == 0 { + return 0, ErrBOF + } + offset = offset - 2 + for { + if offset < 0 { + break + } + b, err := mb.GetByte(offset) + if err != nil { + return 0, err + } + if b == '\n' { + break + } + offset-- + } + return offset + 1, nil +}