skip Files without Id; sort Files by Id

README.md

@@ -64,8 +64,23 @@ Hear goes content. It can be written in html, markdown or whatever what can be p
 
 There are two main commands: `PopulateTree` to prepare source (format, add metadata and add Id) and separate images and other data files and `BuildTree` to get object of tree.
 
-Also both commands use `Config` object that can be prepared by `NewConfig` command. With it can be configurable readable formats, attachment directory, custom meta and excludes (not movable files by regexp).
+Also both commands use `Config` object that can be prepared by `NewConfig` command. With it can be configurable readable formats, attachment directory, custom meta, excludes (not movable files by regexp) and list of functions to apply before file formatting (read bellow).
 
 `Tree` object has methods: `FileById` to get `File` by Id, `Slice` to get sub-tree of given path and `Filter` to filter tree by filter.
 Filter contains array of meta key and searching value. `tag` key is searched as equal and other meta values of keys can contain part. If `-` is before key - filter is filter out.
 
+By full file path and config meta there is possibility to get file params: `Id`, `Meta` and `Tags`:
+
+```
+GetFileParams(fullPath string, meta []string) (string, []string, map[string]string, error)
+```
+
+### Functions to apply before file formatting
+
+There is possibility to add custom functions to apply before file formatting. That is passed with custom function what returns string.
+
+```
+func(dir string, content string, info os.FileInfo) (string, error)
+```
+
+`dir` is path to directory of file, `content` is file content and `info` is file info.

go.mod

@@ -3,7 +3,7 @@ module g.arns.lt/zordsdavini/zord-tree
 go 1.21
 
 require (
-	g.arns.lt/zordsdavini/abcex/v3 v3.0.0
+	g.arns.lt/zordsdavini/abcex/v4 v4.0.4
 	github.com/otiai10/copy v1.6.0
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
 )

go.sum

@@ -1,5 +1,5 @@
-g.arns.lt/zordsdavini/abcex/v3 v3.0.0 h1:LnlDtaBBOpgicMI/dcBUxlu4r4+IFt/MD19JemphgFc=
-g.arns.lt/zordsdavini/abcex/v3 v3.0.0/go.mod h1:z2xqDlRFVnLMnCGpqRjbs9T9EY6lJKJnQmU3zJLSNq8=
+g.arns.lt/zordsdavini/abcex/v4 v4.0.4 h1:idjvgkCjrjZfDxLyOcX7lCIdIndISDAkj77VCvhu8/c=
+g.arns.lt/zordsdavini/abcex/v4 v4.0.4/go.mod h1:/+//gYSUtJrdsmTtWNoffRO4xD1BuPRUMGW4ynet7iE=
 github.com/otiai10/copy v1.6.0 h1:IinKAryFFuPONZ7cm6T6E2QX/vcJwSnlaA5lfoaXIiQ=
 github.com/otiai10/copy v1.6.0/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=

tree.go

@@ -3,6 +3,7 @@ package zord_tree
 import (
 	"bufio"
 	"bytes"
+	"cmp"
 	"crypto/md5"
 	"errors"
 	"fmt"
@@ -13,10 +14,10 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"strings"
 
-	abcex "g.arns.lt/zordsdavini/abcex/v3"
-	"golang.org/x/exp/slices"
+	abcex "g.arns.lt/zordsdavini/abcex/v4"
 )
 
 type Config struct {
@@ -24,8 +25,11 @@ type Config struct {
 	AttachmentDirName string
 	CustomMeta        map[string]func() string
 	Excludes          []string
+	Apps              map[string]AppCallback
 }
 
+type AppCallback func(dir string, content string, info os.FileInfo) (string, error)
+
 type File struct {
 	Id       string
 	Name     string
@@ -147,12 +151,14 @@ func NewConfig(
 	attachmentDirName string,
 	customMeta map[string]func() string,
 	excludes []string,
+	apps map[string]AppCallback,
 ) Config {
 	return Config{
 		ReadableFormats:   readableFormats,
 		AttachmentDirName: attachmentDirName,
 		CustomMeta:        customMeta,
 		Excludes:          excludes,
+		Apps:              apps,
 	}
 }
 
@@ -164,6 +170,13 @@ func PopulateTree(sourcePath string, meta []string, config Config) error {
 	var err error
 	var attachmentRegistry map[string]string
 
+	for _, app := range config.Apps {
+		err = applyApp(app, sourcePath, config)
+		if err != nil {
+			return err
+		}
+	}
+
 	attachmentRegistry, err = moveAttachments(sourcePath, config)
 	if err != nil {
 		return err
@@ -250,6 +263,57 @@ func moveAttachments(dir string, config Config) (map[string]string, error) {
 	return attachmentRegistry, err
 }
 
+func applyApp(app AppCallback, dir string, config Config) error {
+	err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error {
+		if e != nil {
+			return e
+		}
+
+		if info.Mode().IsRegular() && slices.Contains(config.ReadableFormats, path.Ext(fullPath)) {
+			osf, err := os.Open(fullPath)
+			if err != nil {
+				return err
+			}
+
+			// remove all empty lines and separate split line
+			content := ""
+			format := true
+			scanner := bufio.NewScanner(osf)
+			scanner.Split(bufio.ScanLines)
+			for scanner.Scan() {
+				line := scanner.Text()
+				if line == "---" {
+					format = false
+				}
+
+				if format {
+					line = strings.Trim(line, " ")
+					if line != "" {
+						content = content + "\n" + line
+					}
+				} else {
+					content = content + "\n" + line
+				}
+			}
+
+			content, err = app(path.Dir(fullPath), content, info)
+			if err != nil {
+				return err
+			}
+
+			data := []byte(content)
+
+			err = os.WriteFile(fullPath, data, 0644)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	return err
+}
+
 func fixFormat(dir string, attachmentRegistry map[string]string, config Config) error {
 	err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error {
 		if e != nil {
@@ -386,7 +450,7 @@ func addMissingId(dir string, id int64) error {
 					break
 				}
 				if line == "---" {
-					err = addMeta(path, "id", abcex.Encode(id))
+					err = addMeta(path, "id", abcex.Encode(id, abcex.BASE36))
 					if err != nil {
 						return err
 					}
@@ -440,7 +504,7 @@ func getMaxId(dir string) (int64, error) {
 
 				if strings.HasPrefix(line, "* id:") {
 					line = strings.TrimPrefix(line, "* id:")
-					i := abcex.Decode(strings.Trim(line, " "))
+					i := abcex.Decode(strings.Trim(line, " "), abcex.BASE36)
 					if i > max {
 						max = i
 					}
@@ -505,7 +569,16 @@ FILE_LOOP:
 		if err != nil {
 			return tree, err
 		}
+		if nextFile.Id == "" {
+			continue
+		}
 		tree.Files = append(tree.Files, nextFile)
+		slices.SortFunc(tree.Files, func(a, b File) int {
+			if len(a.Id) == len(b.Id) {
+				return strings.Compare(a.Id, b.Id)
+			}
+			return cmp.Compare(len(a.Id), len(b.Id))
+		})
 	}
 
 	return tree, nil
@@ -516,13 +589,30 @@ func readFile(file fs.DirEntry, fullPath string, category []string, meta []strin
 		Name:     file.Name(),
 		FullPath: fullPath,
 		Category: category,
-		Meta:     map[string]string{},
 	}
 
+	id, tags, fileMeta, err := GetFileParams(fullPath, meta)
+	if err != nil {
+		return f, err
+	}
+
+	f.Id = id
+	f.Tags = tags
+	f.Meta = fileMeta
+
+	return f, nil
+}
+
+func GetFileParams(fullPath string, meta []string) (string, []string, map[string]string, error) {
+	fileMeta := map[string]string{}
+	tags := []string{}
+	id := ""
+
 	osf, err := os.Open(fullPath)
 	if err != nil {
-		return File{}, err
+		return id, tags, fileMeta, err
 	}
+
 	scanner := bufio.NewScanner(osf)
 	scanner.Split(bufio.ScanLines)
 	for scanner.Scan() {
@@ -534,27 +624,24 @@ func readFile(file fs.DirEntry, fullPath string, category []string, meta []strin
 		if strings.HasPrefix(line, "* tags:") {
 			line = strings.TrimPrefix(line, "* tags:")
 			t := strings.Split(line, ",")
-			tags := []string{}
 			for _, tag := range t {
 				tags = append(tags, strings.Trim(tag, " "))
 			}
-
-			f.Tags = tags
 		}
 		if strings.HasPrefix(line, "* id:") {
 			line = strings.TrimPrefix(line, "* id:")
-			f.Id = strings.Trim(line, " ")
+			id = strings.Trim(line, " ")
 		}
 		for _, option := range meta {
 			if strings.HasPrefix(line, "* "+option) {
 				line = strings.TrimPrefix(line, "* "+option+":")
-				f.Meta[option] = strings.Trim(line, " ")
+				fileMeta[option] = strings.Trim(line, " ")
 			}
 		}
 	}
 	_ = osf.Close()
 
-	return f, nil
+	return id, tags, fileMeta, nil
 }
 
 func ReadFileContent(file File) (string, error) {

tree_test.go

@@ -22,6 +22,7 @@ func prepare(t *testing.T) Config {
 		"__a",
 		make(map[string]func() string),
 		[]string{"exclude_[0-9]+.bin"},
+		map[string]AppCallback{},
 	)
 
 	err = PopulateTree("./testdata/sunny1", []string{}, config)
@@ -38,6 +39,7 @@ func TestFromNotExistingDirectory(t *testing.T) {
 		"__a",
 		make(map[string]func() string),
 		[]string{},
+		map[string]AppCallback{},
 	)
 
 	_, err := BuildTree("./testing/i_dont_exist", []string{}, config)