package zord_tree import ( "bufio" "bytes" "crypto/md5" "errors" "fmt" "io" "io/fs" "net/url" "os" "path" "path/filepath" "regexp" "strings" abcex "g.arns.lt/zordsdavini/abcex/v4" "golang.org/x/exp/slices" ) type Config struct { ReadableFormats []string AttachmentDirName string CustomMeta map[string]func() string Excludes []string Apps map[string]AppCallback } type AppCallback func(dir string, content string, info os.FileInfo) (string, error) type File struct { Id string Name string FullPath string Category []string Tags []string Meta map[string]string } type Tree struct { Path string Dirs []Tree Files []File } func (t Tree) FileById(id string) (File, error) { for _, f := range t.Files { if f.Id == id { return f, nil } } for _, t2 := range t.Dirs { f, err := t2.FileById(id) if err == nil { return f, nil } } return File{}, errors.New("file was not found") } func (t Tree) Slice(path string) (Tree, error) { if t.Path == path { return t, nil } for _, t2 := range t.Dirs { t3, err := t2.Slice(path) if err == nil { return t3, nil } } return Tree{}, errors.New("tree was not found") } func (t Tree) Filter(filter map[string][]string) (Tree, bool) { filtered := Tree{} filtered.Path = t.Path found := false for _, f := range t.Files { addFile := false exclude := false for option, values := range filter { negative := false if string(option[0]) == "-" { negative = true option = option[1:] } for _, value := range values { if !negative && option == "tag" { for _, tag := range f.Tags { if tag == value { addFile = true } } continue } if negative && option == "tag" { for _, tag := range f.Tags { if tag == value { exclude = true } } continue } if negative && option == "category" { for _, category := range f.Category { if category == value { exclude = true } } continue } if !negative && strings.Contains(f.Meta[option], value) { addFile = true continue } if negative && strings.Contains(f.Meta[option], value) { exclude = true } } if negative && !exclude { addFile = true } } if addFile && !exclude { found = true filtered.Files = append(filtered.Files, f) } } for _, t2 := range t.Dirs { filteredChild, foundChild := t2.Filter(filter) if foundChild { found = true filtered.Dirs = append(filtered.Dirs, filteredChild) } } return filtered, found } func NewConfig( readableFormats []string, attachmentDirName string, customMeta map[string]func() string, excludes []string, apps map[string]AppCallback, ) Config { return Config{ ReadableFormats: readableFormats, AttachmentDirName: attachmentDirName, CustomMeta: customMeta, Excludes: excludes, Apps: apps, } } func BuildTree(dirPath string, meta []string, config Config) (Tree, error) { return readPath(dirPath, []string{}, meta, config) } func PopulateTree(sourcePath string, meta []string, config Config) error { var err error var attachmentRegistry map[string]string for _, app := range config.Apps { err = applyApp(app, sourcePath, config) if err != nil { return err } } attachmentRegistry, err = moveAttachments(sourcePath, config) if err != nil { return err } err = fixFormat(sourcePath, attachmentRegistry, config) if err != nil { return err } var id int64 = 0 id, err = getMaxId(sourcePath) if err != nil { return err } id++ err = addMissingId(sourcePath, id) if err != nil { return err } err = addMissingMeta(sourcePath, meta, config.CustomMeta) return err } func moveAttachments(dir string, config Config) (map[string]string, error) { attachmentRegistry := make(map[string]string) err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error { if e != nil { return e } clearDir := strings.TrimPrefix(dir, "./") if info.Mode().IsRegular() { for _, pattern := range config.Excludes { matched, err := regexp.Match(pattern, []byte(fullPath)) if err == nil && matched { return nil } } } if info.Mode().IsRegular() && !slices.Contains(config.ReadableFormats, path.Ext(fullPath)) && !strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, config.AttachmentDirName)) { f, err := os.Open(fullPath) if err != nil { return err } defer f.Close() h := md5.New() if _, err := io.Copy(h, f); err != nil { return err } _ = os.Mkdir(fmt.Sprintf("%s/%s", dir, config.AttachmentDirName), 0755) attachmentRegistry[fullPath] = fmt.Sprintf( "%s/%x%s", config.AttachmentDirName, h.Sum(nil), path.Ext(fullPath), ) newPath := fmt.Sprintf( "%s/%s/%x%s", dir, config.AttachmentDirName, h.Sum(nil), path.Ext(fullPath), ) err = os.Rename(fullPath, newPath) if err != nil { return err } } return nil }) return attachmentRegistry, err } func applyApp(app AppCallback, dir string, config Config) error { err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error { if e != nil { return e } if info.Mode().IsRegular() && slices.Contains(config.ReadableFormats, path.Ext(fullPath)) { osf, err := os.Open(fullPath) if err != nil { return err } // remove all empty lines and separate split line content := "" format := true scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() if line == "---" { format = false } if format { line = strings.Trim(line, " ") if line != "" { content = content + "\n" + line } } else { content = content + "\n" + line } } content, err = app(path.Dir(fullPath), content, info) if err != nil { return err } data := []byte(content) err = os.WriteFile(fullPath, data, 0644) if err != nil { return err } } return nil }) return err } func fixFormat(dir string, attachmentRegistry map[string]string, config Config) error { err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error { if e != nil { return e } if info.Mode().IsRegular() && slices.Contains(config.ReadableFormats, path.Ext(fullPath)) { osf, err := os.Open(fullPath) if err != nil { return err } // remove all empty lines and separate split line content := "" format := true scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() if line == "---" { format = false } if format { line = strings.Trim(line, " ") if line != "" { content = content + "\n" + line } } else { content = content + "\n" + line } } // fix attachments data := []byte(content) re := regexp.MustCompile(`!?\[([^\]*]*)\]\(([^\) ]*)\)`) for _, match := range re.FindAllSubmatch(data, -1) { _, err := url.ParseRequestURI(string(match[2][:])) if err != nil { aPath := path.Clean( fmt.Sprintf("%s/%s", path.Dir(fullPath), string(match[2][:])), ) if _, ok := attachmentRegistry[aPath]; ok { link := fmt.Sprintf("![%s](%s)", match[1], attachmentRegistry[aPath]) data = bytes.Replace(data, match[0], []byte(link), 1) } } } err = os.WriteFile(fullPath, data, 0644) // format split line b, err := os.ReadFile(fullPath) // just pass the file name if err != nil { return err } str := string(b) str = strings.Replace(str, "\n---\n", "\n\n---\n", 1) err = os.WriteFile(fullPath, []byte(str), 0644) } return nil }) return err } func addMissingMeta(dir string, meta []string, customMeta map[string]func() string) error { err := filepath.Walk(dir, func(path string, info os.FileInfo, e error) error { if e != nil { return e } if info.Mode().IsRegular() { check := map[string]bool{} for _, option := range meta { check[option] = false } osf, err := os.Open(path) if err != nil { return err } scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() for _, option := range meta { if strings.HasPrefix(line, "* "+option+":") { check[option] = true } } if line == "---" { for option, process := range check { if !process { defaultValue := "" if _, ok := customMeta[option]; ok { defaultValue = customMeta[option]() } err = addMeta(path, option, defaultValue) if err != nil { return err } } } break } } } return nil }) return err } func addMissingId(dir string, id int64) error { err := filepath.Walk(dir, func(path string, info os.FileInfo, e error) error { if e != nil { return e } if info.Mode().IsRegular() { osf, err := os.Open(path) if err != nil { return err } scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() if strings.HasPrefix(line, "* id:") { break } if line == "---" { err = addMeta(path, "id", abcex.Encode(id, abcex.BASE36)) if err != nil { return err } id++ break } } } return nil }) if err != nil { return err } return nil } func addMeta(path string, option string, value string) error { b, err := os.ReadFile(path) // just pass the file name if err != nil { return err } str := string(b) str = strings.Replace(str, "\n\n---\n", fmt.Sprintf("\n* %s: %s\n\n---\n", option, value), 1) err = os.WriteFile(path, []byte(str), 0644) return err } func getMaxId(dir string) (int64, error) { var max int64 = 0 err := filepath.Walk(dir, func(path string, info os.FileInfo, e error) error { if e != nil { return e } if info.Mode().IsRegular() { osf, err := os.Open(path) if err != nil { return err } scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() if line == "---" { break } if strings.HasPrefix(line, "* id:") { line = strings.TrimPrefix(line, "* id:") i := abcex.Decode(strings.Trim(line, " "), abcex.BASE36) if i > max { max = i } } } } return nil }) if err != nil { return max, err } return max, nil } func readPath(dirPath string, category []string, meta []string, config Config) (Tree, error) { clearDir := strings.TrimPrefix(dirPath, "./") tree := Tree{} tree.Path = clearDir files, err := os.ReadDir(dirPath) if err != nil { return tree, err } FILE_LOOP: for _, file := range files { if strings.HasPrefix(file.Name(), ".") { continue } fullPath := path.Join(dirPath, file.Name()) if file.IsDir() { if strings.HasPrefix( fullPath, fmt.Sprintf("%s/%s", clearDir, config.AttachmentDirName), ) { continue } nextDir, err := readPath(fullPath, append(category, file.Name()), meta, config) if err != nil { return tree, err } tree.Dirs = append(tree.Dirs, nextDir) continue } else { for _, pattern := range config.Excludes { matched, err := regexp.Match(pattern, []byte(fullPath)) if err == nil && matched { continue FILE_LOOP } } } _, err := os.ReadFile(fullPath) if err != nil { return tree, err } nextFile, err := readFile(file, fullPath, category, meta) if err != nil { return tree, err } tree.Files = append(tree.Files, nextFile) } return tree, nil } func readFile(file fs.DirEntry, fullPath string, category []string, meta []string) (File, error) { f := File{ Name: file.Name(), FullPath: fullPath, Category: category, } id, tags, fileMeta, err := GetFileParams(fullPath, meta) if err != nil { return f, err } f.Id = id f.Tags = tags f.Meta = fileMeta return f, nil } func GetFileParams(fullPath string, meta []string) (string, []string, map[string]string, error) { fileMeta := map[string]string{} tags := []string{} id := "" osf, err := os.Open(fullPath) if err != nil { return id, tags, fileMeta, err } scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() if line == "---" { break } if strings.HasPrefix(line, "* tags:") { line = strings.TrimPrefix(line, "* tags:") t := strings.Split(line, ",") for _, tag := range t { tags = append(tags, strings.Trim(tag, " ")) } } if strings.HasPrefix(line, "* id:") { line = strings.TrimPrefix(line, "* id:") id = strings.Trim(line, " ") } for _, option := range meta { if strings.HasPrefix(line, "* "+option) { line = strings.TrimPrefix(line, "* "+option+":") fileMeta[option] = strings.Trim(line, " ") } } } _ = osf.Close() return id, tags, fileMeta, nil } func ReadFileContent(file File) (string, error) { osf, err := os.Open(file.FullPath) if err != nil { return "", err } content := "" separator := "" removeEmptyLine := true isMetaPart := true scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() if line == "---" { isMetaPart = false continue } if isMetaPart { continue } if removeEmptyLine { line = strings.Trim(line, " ") if line == "" { continue } else { removeEmptyLine = false } } content = content + separator + line if separator == "" { separator = "\n" } } _ = osf.Close() return content, nil }