package zord_tree import ( "bufio" "bytes" "crypto/md5" "errors" "fmt" "g.arns.lt/zordsdavini/abcex" cp "github.com/otiai10/copy" "golang.org/x/exp/slices" "io" "io/fs" "io/ioutil" "net/url" "os" "path" "path/filepath" "regexp" "strings" ) type File struct { Id string Name string FullPath string Category []string Tags []string Meta map[string]string } type Tree struct { Path string Dirs []Tree Files []File } var readableFormats = []string{".md"} const attachmentDirName = "__a" func (t Tree) FileById(id string) (File, error) { for _, f := range t.Files { if f.Id == id { return f, nil } } for _, t2 := range t.Dirs { f, err := t2.FileById(id) if err == nil { return f, nil } } return File{}, errors.New("file was not found") } func (t Tree) Slice(path string) (Tree, error) { if t.Path == path { return t, nil } for _, t2 := range t.Dirs { t3, err := t2.Slice(path) if err == nil { return t3, nil } } return Tree{}, errors.New("tree was not found") } func (t Tree) Filter(filter map[string][]string) (Tree, bool) { filtered := Tree{} filtered.Path = t.Path found := false for _, f := range t.Files { addFile := false for option, values := range filter { for _, value := range values { if option == "tag" { for _, tag := range f.Tags { if tag == value { addFile = true } } continue } if strings.Contains(f.Meta[option], value) { addFile = true } } } if addFile { found = true filtered.Files = append(filtered.Files, f) } } for _, t2 := range t.Dirs { filteredChild, foundChild := t2.Filter(filter) if foundChild { found = true filtered.Dirs = append(filtered.Dirs, filteredChild) } } return filtered, found } func BuildTree(dirPath string, meta []string) (Tree, error) { return readPath(dirPath, []string{}, meta) } func PopulateTree(sourcePath string, destPath string, meta []string, customMeta map[string]func() string) error { var err error var attachmentRegistry map[string]string err = removeHidden(sourcePath) if err != nil { return err } attachmentRegistry, err = moveAttachments(sourcePath) if err != nil { return err } err = fixFormat(sourcePath, attachmentRegistry) if err != nil { return err } var id int64 = 0 id, err = getMaxId(sourcePath) if err != nil { return err } id++ err = addMissingId(sourcePath, id) if err != nil { return err } err = addMissingMeta(sourcePath, meta, customMeta) if err != nil { return err } err = cp.Copy(sourcePath, destPath) return err } func removeHidden(dir string) error { var toRemovePaths []string err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error { if e != nil { return e } if strings.HasPrefix(info.Name(), ".") { toRemovePaths = append(toRemovePaths, fullPath) } return nil }) for _, removePath := range toRemovePaths { e := os.RemoveAll(removePath) if e != nil { return e } } return err } func moveAttachments(dir string) (map[string]string, error) { attachmentRegistry := make(map[string]string) err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error { if e != nil { return e } var clearDir = strings.TrimPrefix(dir, "./") if info.Mode().IsRegular() && !slices.Contains(readableFormats, path.Ext(fullPath)) && !strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, attachmentDirName)) { f, err := os.Open(fullPath) if err != nil { return err } defer f.Close() h := md5.New() if _, err := io.Copy(h, f); err != nil { return err } _ = os.Mkdir(fmt.Sprintf("%s/%s", dir, attachmentDirName), 0755) attachmentRegistry[fullPath] = fmt.Sprintf("%s/%x%s", attachmentDirName, h.Sum(nil), path.Ext(fullPath)) newPath := fmt.Sprintf("%s/%s/%x%s", dir, attachmentDirName, h.Sum(nil), path.Ext(fullPath)) err = os.Rename(fullPath, newPath) if err != nil { return err } } return nil }) return attachmentRegistry, err } func fixFormat(dir string, attachmentRegistry map[string]string) error { err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error { if e != nil { return e } if info.Mode().IsRegular() && slices.Contains(readableFormats, path.Ext(fullPath)) { osf, err := os.Open(fullPath) if err != nil { return err } // remove all empty lines and separate split line content := "" format := true scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() if line == "---" { format = false } if format { line = strings.Trim(line, " ") if line != "" { content = content + "\n" + line } } else { content = content + "\n" + line } } // fix attachments data := []byte(content) re := regexp.MustCompile(`!?\[([^\]*]*)\]\(([^\) ]*)\)`) for _, match := range re.FindAllSubmatch(data, -1) { _, err := url.ParseRequestURI(string(match[2][:])) if err != nil { aPath := path.Clean(fmt.Sprintf("%s/%s", path.Dir(fullPath), string(match[2][:]))) if _, ok := attachmentRegistry[aPath]; ok { link := fmt.Sprintf("![%s](%s)", match[1], attachmentRegistry[aPath]) data = bytes.Replace(data, match[0], []byte(link), 1) } } } err = ioutil.WriteFile(fullPath, data, 0644) // format split line b, err := ioutil.ReadFile(fullPath) // just pass the file name if err != nil { return err } str := string(b) str = strings.Replace(str, "\n---\n", "\n\n---\n", 1) err = ioutil.WriteFile(fullPath, []byte(str), 0644) } return nil }) return err } func addMissingMeta(dir string, meta []string, customMeta map[string]func() string) error { err := filepath.Walk(dir, func(path string, info os.FileInfo, e error) error { if e != nil { return e } if info.Mode().IsRegular() { check := map[string]bool{} for _, option := range meta { check[option] = false } osf, err := os.Open(path) if err != nil { return err } scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() for _, option := range meta { if strings.HasPrefix(line, "* "+option+":") { check[option] = true } } if line == "---" { for option, process := range check { if !process { defaultValue := "" if _, ok := customMeta[option]; ok { defaultValue = customMeta[option]() } err = addMeta(path, option, defaultValue) if err != nil { return err } } } break } } } return nil }) return err } func addMissingId(dir string, id int64) error { err := filepath.Walk(dir, func(path string, info os.FileInfo, e error) error { if e != nil { return e } if info.Mode().IsRegular() { osf, err := os.Open(path) if err != nil { return err } scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() if strings.HasPrefix(line, "* id:") { break } if line == "---" { err = addMeta(path, "id", abcex.Encode(id)) if err != nil { return err } id++ break } } } return nil }) if err != nil { return err } return nil } func addMeta(path string, option string, value string) error { b, err := ioutil.ReadFile(path) // just pass the file name if err != nil { return err } str := string(b) str = strings.Replace(str, "\n\n---\n", fmt.Sprintf("\n* %s: %s\n\n---\n", option, value), 1) err = ioutil.WriteFile(path, []byte(str), 0644) return err } func getMaxId(dir string) (int64, error) { var max int64 = 0 err := filepath.Walk(dir, func(path string, info os.FileInfo, e error) error { if e != nil { return e } if info.Mode().IsRegular() { osf, err := os.Open(path) if err != nil { return err } scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() if line == "---" { break } if strings.HasPrefix(line, "* id:") { line = strings.TrimPrefix(line, "* id:") i := abcex.Decode(strings.Trim(line, " ")) if i > max { max = i } } } } return nil }) if err != nil { return max, err } return max, nil } func readPath(dirPath string, category []string, meta []string) (Tree, error) { var clearDir = strings.TrimPrefix(dirPath, "./") tree := Tree{} tree.Path = clearDir files, err := ioutil.ReadDir(dirPath) if err != nil { return tree, err } for _, file := range files { fullPath := path.Join(dirPath, file.Name()) if file.IsDir() { if strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, attachmentDirName)) { continue } nextDir, err := readPath(fullPath, append(category, file.Name()), meta) if err != nil { return tree, err } tree.Dirs = append(tree.Dirs, nextDir) continue } _, err := ioutil.ReadFile(fullPath) if err != nil { return tree, err } nextFile, err := readFile(file, fullPath, category, meta) if err != nil { return tree, err } tree.Files = append(tree.Files, nextFile) } return tree, nil } func readFile(file fs.FileInfo, fullPath string, category []string, meta []string) (File, error) { f := File{ Name: file.Name(), FullPath: fullPath, Category: category, Meta: map[string]string{}, } osf, err := os.Open(fullPath) if err != nil { return File{}, err } scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() if line == "---" { break } if strings.HasPrefix(line, "* tags:") { line = strings.TrimPrefix(line, "* tags:") t := strings.Split(line, ",") tags := []string{} for _, tag := range t { tags = append(tags, strings.Trim(tag, " ")) } f.Tags = tags } if strings.HasPrefix(line, "* id:") { line = strings.TrimPrefix(line, "* id:") f.Id = strings.Trim(line, " ") } for _, option := range meta { if strings.HasPrefix(line, "* "+option) { line = strings.TrimPrefix(line, "* "+option+":") f.Meta[option] = strings.Trim(line, " ") } } } _ = osf.Close() return f, nil } func ReadFileContent(file File) (string, error) { osf, err := os.Open(file.FullPath) if err != nil { return "", err } content := "" separator := "" removeEmptyLine := true isMetaPart := true scanner := bufio.NewScanner(osf) scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() if line == "---" { isMetaPart = false continue } if isMetaPart { continue } if removeEmptyLine { line = strings.Trim(line, " ") if line == "" { continue } else { removeEmptyLine = false } } content = content + separator + line if separator == "" { separator = "\n" } } _ = osf.Close() return content, nil }