zord-tree/tree.go

package zord_tree

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	abcex "g.arns.lt/zordsdavini/abcex/v4"
	"golang.org/x/exp/slices"
)

type Config struct {
	ReadableFormats   []string
	AttachmentDirName string
	CustomMeta        map[string]func() string
	Excludes          []string
	Apps              map[string]AppCallback
}

type AppCallback func(dir string, content string, info os.FileInfo) (string, error)

type File struct {
	Id       string
	Name     string
	FullPath string
	Category []string
	Tags     []string
	Meta     map[string]string
}

type Tree struct {
	Path  string
	Dirs  []Tree
	Files []File
}

func (t Tree) FileById(id string) (File, error) {
	for _, f := range t.Files {
		if f.Id == id {
			return f, nil
		}
	}
	for _, t2 := range t.Dirs {
		f, err := t2.FileById(id)
		if err == nil {
			return f, nil
		}
	}

	return File{}, errors.New("file was not found")
}

func (t Tree) Slice(path string) (Tree, error) {
	if t.Path == path {
		return t, nil
	}

	for _, t2 := range t.Dirs {
		t3, err := t2.Slice(path)
		if err == nil {
			return t3, nil
		}
	}

	return Tree{}, errors.New("tree was not found")
}

func (t Tree) Filter(filter map[string][]string) (Tree, bool) {
	filtered := Tree{}
	filtered.Path = t.Path
	found := false

	for _, f := range t.Files {
		addFile := false
		exclude := false
		for option, values := range filter {
			negative := false
			if string(option[0]) == "-" {
				negative = true
				option = option[1:]
			}

			for _, value := range values {
				if !negative && option == "tag" {
					for _, tag := range f.Tags {
						if tag == value {
							addFile = true
						}
					}
					continue
				}
				if negative && option == "tag" {
					for _, tag := range f.Tags {
						if tag == value {
							exclude = true
						}
					}
					continue
				}
				if negative && option == "category" {
					for _, category := range f.Category {
						if category == value {
							exclude = true
						}
					}
					continue
				}
				if !negative && strings.Contains(f.Meta[option], value) {
					addFile = true
					continue
				}
				if negative && strings.Contains(f.Meta[option], value) {
					exclude = true
				}
			}

			if negative && !exclude {
				addFile = true
			}
		}
		if addFile && !exclude {
			found = true
			filtered.Files = append(filtered.Files, f)
		}
	}

	for _, t2 := range t.Dirs {
		filteredChild, foundChild := t2.Filter(filter)
		if foundChild {
			found = true
			filtered.Dirs = append(filtered.Dirs, filteredChild)
		}
	}

	return filtered, found
}

func NewConfig(
	readableFormats []string,
	attachmentDirName string,
	customMeta map[string]func() string,
	excludes []string,
	apps map[string]AppCallback,
) Config {
	return Config{
		ReadableFormats:   readableFormats,
		AttachmentDirName: attachmentDirName,
		CustomMeta:        customMeta,
		Excludes:          excludes,
		Apps:              apps,
	}
}

func BuildTree(dirPath string, meta []string, config Config) (Tree, error) {
	return readPath(dirPath, []string{}, meta, config)
}

func PopulateTree(sourcePath string, meta []string, config Config) error {
	var err error
	var attachmentRegistry map[string]string

	for _, app := range config.Apps {
		err = applyApp(app, sourcePath, config)
		if err != nil {
			return err
		}
	}

	attachmentRegistry, err = moveAttachments(sourcePath, config)
	if err != nil {
		return err
	}

	err = fixFormat(sourcePath, attachmentRegistry, config)
	if err != nil {
		return err
	}

	var id int64 = 0
	id, err = getMaxId(sourcePath)
	if err != nil {
		return err
	}

	id++
	err = addMissingId(sourcePath, id)
	if err != nil {
		return err
	}

	err = addMissingMeta(sourcePath, meta, config.CustomMeta)

	return err
}

func moveAttachments(dir string, config Config) (map[string]string, error) {
	attachmentRegistry := make(map[string]string)

	err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error {
		if e != nil {
			return e
		}

		clearDir := strings.TrimPrefix(dir, "./")
		if info.Mode().IsRegular() {
			for _, pattern := range config.Excludes {
				matched, err := regexp.Match(pattern, []byte(fullPath))
				if err == nil && matched {
					return nil
				}
			}
		}

		if info.Mode().IsRegular() &&
			!slices.Contains(config.ReadableFormats, path.Ext(fullPath)) &&
			!strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, config.AttachmentDirName)) {
			f, err := os.Open(fullPath)
			if err != nil {
				return err
			}
			defer f.Close()

			h := md5.New()
			if _, err := io.Copy(h, f); err != nil {
				return err
			}

			_ = os.Mkdir(fmt.Sprintf("%s/%s", dir, config.AttachmentDirName), 0755)

			attachmentRegistry[fullPath] = fmt.Sprintf(
				"%s/%x%s",
				config.AttachmentDirName,
				h.Sum(nil),
				path.Ext(fullPath),
			)
			newPath := fmt.Sprintf(
				"%s/%s/%x%s",
				dir,
				config.AttachmentDirName,
				h.Sum(nil),
				path.Ext(fullPath),
			)
			err = os.Rename(fullPath, newPath)
			if err != nil {
				return err
			}
		}

		return nil
	})

	return attachmentRegistry, err
}

func applyApp(app AppCallback, dir string, config Config) error {
	err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error {
		if e != nil {
			return e
		}

		if info.Mode().IsRegular() && slices.Contains(config.ReadableFormats, path.Ext(fullPath)) {
			osf, err := os.Open(fullPath)
			if err != nil {
				return err
			}

			// remove all empty lines and separate split line
			content := ""
			format := true
			scanner := bufio.NewScanner(osf)
			scanner.Split(bufio.ScanLines)
			for scanner.Scan() {
				line := scanner.Text()
				if line == "---" {
					format = false
				}

				if format {
					line = strings.Trim(line, " ")
					if line != "" {
						content = content + "\n" + line
					}
				} else {
					content = content + "\n" + line
				}
			}

			content, err = app(path.Dir(fullPath), content, info)
			if err != nil {
				return err
			}

			data := []byte(content)

			err = os.WriteFile(fullPath, data, 0644)
			if err != nil {
				return err
			}
		}
		return nil
	})

	return err
}

func fixFormat(dir string, attachmentRegistry map[string]string, config Config) error {
	err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error {
		if e != nil {
			return e
		}

		if info.Mode().IsRegular() && slices.Contains(config.ReadableFormats, path.Ext(fullPath)) {
			osf, err := os.Open(fullPath)
			if err != nil {
				return err
			}

			// remove all empty lines and separate split line
			content := ""
			format := true
			scanner := bufio.NewScanner(osf)
			scanner.Split(bufio.ScanLines)
			for scanner.Scan() {
				line := scanner.Text()
				if line == "---" {
					format = false
				}

				if format {
					line = strings.Trim(line, " ")
					if line != "" {
						content = content + "\n" + line
					}
				} else {
					content = content + "\n" + line
				}
			}

			// fix attachments
			data := []byte(content)
			re := regexp.MustCompile(`!?\[([^\]*]*)\]\(([^\) ]*)\)`)

			for _, match := range re.FindAllSubmatch(data, -1) {
				_, err := url.ParseRequestURI(string(match[2][:]))
				if err != nil {
					aPath := path.Clean(
						fmt.Sprintf("%s/%s", path.Dir(fullPath), string(match[2][:])),
					)
					if _, ok := attachmentRegistry[aPath]; ok {
						link := fmt.Sprintf("![%s](%s)", match[1], attachmentRegistry[aPath])
						data = bytes.Replace(data, match[0], []byte(link), 1)
					}
				}
			}

			err = os.WriteFile(fullPath, data, 0644)

			// format split line
			b, err := os.ReadFile(fullPath) // just pass the file name
			if err != nil {
				return err
			}

			str := string(b)
			str = strings.Replace(str, "\n---\n", "\n\n---\n", 1)
			err = os.WriteFile(fullPath, []byte(str), 0644)
		}
		return nil
	})

	return err
}

func addMissingMeta(dir string, meta []string, customMeta map[string]func() string) error {
	err := filepath.Walk(dir, func(path string, info os.FileInfo, e error) error {
		if e != nil {
			return e
		}

		if info.Mode().IsRegular() {
			check := map[string]bool{}
			for _, option := range meta {
				check[option] = false
			}

			osf, err := os.Open(path)
			if err != nil {
				return err
			}
			scanner := bufio.NewScanner(osf)
			scanner.Split(bufio.ScanLines)
			for scanner.Scan() {
				line := scanner.Text()
				for _, option := range meta {
					if strings.HasPrefix(line, "* "+option+":") {
						check[option] = true
					}
				}
				if line == "---" {
					for option, process := range check {
						if !process {
							defaultValue := ""
							if _, ok := customMeta[option]; ok {
								defaultValue = customMeta[option]()
							}
							err = addMeta(path, option, defaultValue)
							if err != nil {
								return err
							}
						}
					}

					break
				}
			}
		}
		return nil
	})

	return err
}

func addMissingId(dir string, id int64) error {
	err := filepath.Walk(dir, func(path string, info os.FileInfo, e error) error {
		if e != nil {
			return e
		}

		if info.Mode().IsRegular() {
			osf, err := os.Open(path)
			if err != nil {
				return err
			}
			scanner := bufio.NewScanner(osf)
			scanner.Split(bufio.ScanLines)
			for scanner.Scan() {
				line := scanner.Text()
				if strings.HasPrefix(line, "* id:") {
					break
				}
				if line == "---" {
					err = addMeta(path, "id", abcex.Encode(id, abcex.BASE36))
					if err != nil {
						return err
					}

					id++
					break
				}
			}
		}
		return nil
	})
	if err != nil {
		return err
	}

	return nil
}

func addMeta(path string, option string, value string) error {
	b, err := os.ReadFile(path) // just pass the file name
	if err != nil {
		return err
	}

	str := string(b)
	str = strings.Replace(str, "\n\n---\n", fmt.Sprintf("\n* %s: %s\n\n---\n", option, value), 1)
	err = os.WriteFile(path, []byte(str), 0644)

	return err
}

func getMaxId(dir string) (int64, error) {
	var max int64 = 0
	err := filepath.Walk(dir, func(path string, info os.FileInfo, e error) error {
		if e != nil {
			return e
		}

		if info.Mode().IsRegular() {
			osf, err := os.Open(path)
			if err != nil {
				return err
			}
			scanner := bufio.NewScanner(osf)
			scanner.Split(bufio.ScanLines)
			for scanner.Scan() {
				line := scanner.Text()
				if line == "---" {
					break
				}

				if strings.HasPrefix(line, "* id:") {
					line = strings.TrimPrefix(line, "* id:")
					i := abcex.Decode(strings.Trim(line, " "), abcex.BASE36)
					if i > max {
						max = i
					}
				}
			}
		}
		return nil
	})
	if err != nil {
		return max, err
	}

	return max, nil
}

func readPath(dirPath string, category []string, meta []string, config Config) (Tree, error) {
	clearDir := strings.TrimPrefix(dirPath, "./")

	tree := Tree{}
	tree.Path = clearDir

	files, err := os.ReadDir(dirPath)
	if err != nil {
		return tree, err
	}

FILE_LOOP:
	for _, file := range files {
		if strings.HasPrefix(file.Name(), ".") {
			continue
		}

		fullPath := path.Join(dirPath, file.Name())
		if file.IsDir() {
			if strings.HasPrefix(
				fullPath,
				fmt.Sprintf("%s/%s", clearDir, config.AttachmentDirName),
			) {
				continue
			}

			nextDir, err := readPath(fullPath, append(category, file.Name()), meta, config)
			if err != nil {
				return tree, err
			}
			tree.Dirs = append(tree.Dirs, nextDir)
			continue
		} else {
			for _, pattern := range config.Excludes {
				matched, err := regexp.Match(pattern, []byte(fullPath))
				if err == nil && matched {
					continue FILE_LOOP
				}
			}
		}

		_, err := os.ReadFile(fullPath)
		if err != nil {
			return tree, err
		}
		nextFile, err := readFile(file, fullPath, category, meta)
		if err != nil {
			return tree, err
		}
		tree.Files = append(tree.Files, nextFile)
	}

	return tree, nil
}

func readFile(file fs.DirEntry, fullPath string, category []string, meta []string) (File, error) {
	f := File{
		Name:     file.Name(),
		FullPath: fullPath,
		Category: category,
	}

	id, tags, fileMeta, err := GetFileParams(fullPath, meta)
	if err != nil {
		return f, err
	}

	f.Id = id
	f.Tags = tags
	f.Meta = fileMeta

	return f, nil
}

func GetFileParams(fullPath string, meta []string) (string, []string, map[string]string, error) {
	fileMeta := map[string]string{}
	tags := []string{}
	id := ""

	osf, err := os.Open(fullPath)
	if err != nil {
		return id, tags, fileMeta, err
	}

	scanner := bufio.NewScanner(osf)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		if line == "---" {
			break
		}

		if strings.HasPrefix(line, "* tags:") {
			line = strings.TrimPrefix(line, "* tags:")
			t := strings.Split(line, ",")
			for _, tag := range t {
				tags = append(tags, strings.Trim(tag, " "))
			}
		}
		if strings.HasPrefix(line, "* id:") {
			line = strings.TrimPrefix(line, "* id:")
			id = strings.Trim(line, " ")
		}
		for _, option := range meta {
			if strings.HasPrefix(line, "* "+option) {
				line = strings.TrimPrefix(line, "* "+option+":")
				fileMeta[option] = strings.Trim(line, " ")
			}
		}
	}
	_ = osf.Close()

	return id, tags, fileMeta, nil
}

func ReadFileContent(file File) (string, error) {
	osf, err := os.Open(file.FullPath)
	if err != nil {
		return "", err
	}

	content := ""
	separator := ""
	removeEmptyLine := true
	isMetaPart := true
	scanner := bufio.NewScanner(osf)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		if line == "---" {
			isMetaPart = false
			continue
		}

		if isMetaPart {
			continue
		}

		if removeEmptyLine {
			line = strings.Trim(line, " ")
			if line == "" {
				continue
			} else {
				removeEmptyLine = false
			}
		}

		content = content + separator + line
		if separator == "" {
			separator = "\n"
		}
	}
	_ = osf.Close()

	return content, nil
}