added excludes, main Config object

README.md

@@ -8,7 +8,7 @@ content. For now it supports markdown format.
 Process should go in two steps:
 
 * populate existing `source` directory with formatting files, add missing `id` and meta data
-* separate images and other data files to `__a` root directory
+* separate images and other data files to ex. `__a` root directory
 * build `Tree` to operate on object for your app
 * read file content without meta data by path
 
@@ -64,6 +64,8 @@ Hear goes content. It can be written in html, markdown or whatever what can be p
 
 There are two main commands: `PopulateTree` to prepare source (format, add metadata and add Id) and separate images and other data files and `BuildTree` to get object of tree.
 
+Also both commands use `Config` object that can be prepared by `NewConfig` command. With it can be configurable readable formats, attachment directory, custom meta and excludes (not movable files by regexp).
+
 `Tree` object has methods: `FileById` to get `File` by Id, `Slice` to get sub-tree of given path and `Filter` to filter tree by filter.
 Filter contains array of meta key and searching value. `tag` key is searched as equal and other meta values of keys can contain part.
 

go.mod

@@ -1,6 +1,6 @@
 module g.arns.lt/zordsdavini/zord-tree
 
-go 1.18
+go 1.21
 
 require (
 	g.arns.lt/zordsdavini/abcex v1.0.0

tree.go

@@ -6,19 +6,26 @@ import (
 	"crypto/md5"
 	"errors"
 	"fmt"
-	"g.arns.lt/zordsdavini/abcex"
-	"golang.org/x/exp/slices"
 	"io"
 	"io/fs"
-	"io/ioutil"
 	"net/url"
 	"os"
 	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
+
+	"g.arns.lt/zordsdavini/abcex"
+	"golang.org/x/exp/slices"
 )
 
+type Config struct {
+	ReadableFormats   []string
+	AttachmentDirName string
+	CustomMeta        map[string]func() string
+	Excludes          []string
+}
+
 type File struct {
 	Id       string
 	Name     string
@@ -34,10 +41,6 @@ type Tree struct {
 	Files []File
 }
 
-var readableFormats = []string{".md"}
-
-const AttachmentDirName = "__a"
-
 func (t Tree) FileById(id string) (File, error) {
 	for _, f := range t.Files {
 		if f.Id == id {
@@ -108,20 +111,34 @@ func (t Tree) Filter(filter map[string][]string) (Tree, bool) {
 	return filtered, found
 }
 
-func BuildTree(dirPath string, meta []string) (Tree, error) {
+func NewConfig(
-	return readPath(dirPath, []string{}, meta)
+	readableFormats []string,
+	attachmentDirName string,
+	customMeta map[string]func() string,
+	excludes []string,
+) Config {
+	return Config{
+		ReadableFormats:   readableFormats,
+		AttachmentDirName: attachmentDirName,
+		CustomMeta:        customMeta,
+		Excludes:          excludes,
+	}
 }
 
-func PopulateTree(sourcePath string, meta []string, customMeta map[string]func() string) error {
+func BuildTree(dirPath string, meta []string, config Config) (Tree, error) {
+	return readPath(dirPath, []string{}, meta, config)
+}
+
+func PopulateTree(sourcePath string, meta []string, config Config) error {
 	var err error
 	var attachmentRegistry map[string]string
 
-	attachmentRegistry, err = moveAttachments(sourcePath)
+	attachmentRegistry, err = moveAttachments(sourcePath, config)
 	if err != nil {
 		return err
 	}
 
-	err = fixFormat(sourcePath, attachmentRegistry)
+	err = fixFormat(sourcePath, attachmentRegistry, config)
 	if err != nil {
 		return err
 	}
@@ -138,12 +155,12 @@ func PopulateTree(sourcePath string, meta []string, customMeta map[string]func()
 		return err
 	}
 
-	err = addMissingMeta(sourcePath, meta, customMeta)
+	err = addMissingMeta(sourcePath, meta, config.CustomMeta)
 
 	return err
 }
 
-func moveAttachments(dir string) (map[string]string, error) {
+func moveAttachments(dir string, config Config) (map[string]string, error) {
 	attachmentRegistry := make(map[string]string)
 
 	err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error {
@@ -151,8 +168,19 @@ func moveAttachments(dir string) (map[string]string, error) {
 			return e
 		}
 
-		var clearDir = strings.TrimPrefix(dir, "./")
+		clearDir := strings.TrimPrefix(dir, "./")
-		if info.Mode().IsRegular() && !slices.Contains(readableFormats, path.Ext(fullPath)) && !strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, AttachmentDirName)) {
+		if info.Mode().IsRegular() {
+			for _, pattern := range config.Excludes {
+				matched, err := regexp.Match(pattern, []byte(fullPath))
+				if err == nil && matched {
+					return nil
+				}
+			}
+		}
+
+		if info.Mode().IsRegular() &&
+			!slices.Contains(config.ReadableFormats, path.Ext(fullPath)) &&
+			!strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, config.AttachmentDirName)) {
 			f, err := os.Open(fullPath)
 			if err != nil {
 				return err
@@ -164,10 +192,21 @@ func moveAttachments(dir string) (map[string]string, error) {
 				return err
 			}
 
-			_ = os.Mkdir(fmt.Sprintf("%s/%s", dir, AttachmentDirName), 0755)
+			_ = os.Mkdir(fmt.Sprintf("%s/%s", dir, config.AttachmentDirName), 0755)
 
-			attachmentRegistry[fullPath] = fmt.Sprintf("%s/%x%s", AttachmentDirName, h.Sum(nil), path.Ext(fullPath))
+			attachmentRegistry[fullPath] = fmt.Sprintf(
-			newPath := fmt.Sprintf("%s/%s/%x%s", dir, AttachmentDirName, h.Sum(nil), path.Ext(fullPath))
+				"%s/%x%s",
+				config.AttachmentDirName,
+				h.Sum(nil),
+				path.Ext(fullPath),
+			)
+			newPath := fmt.Sprintf(
+				"%s/%s/%x%s",
+				dir,
+				config.AttachmentDirName,
+				h.Sum(nil),
+				path.Ext(fullPath),
+			)
 			err = os.Rename(fullPath, newPath)
 			if err != nil {
 				return err
@@ -180,13 +219,13 @@ func moveAttachments(dir string) (map[string]string, error) {
 	return attachmentRegistry, err
 }
 
-func fixFormat(dir string, attachmentRegistry map[string]string) error {
+func fixFormat(dir string, attachmentRegistry map[string]string, config Config) error {
 	err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error {
 		if e != nil {
 			return e
 		}
 
-		if info.Mode().IsRegular() && slices.Contains(readableFormats, path.Ext(fullPath)) {
+		if info.Mode().IsRegular() && slices.Contains(config.ReadableFormats, path.Ext(fullPath)) {
 			osf, err := os.Open(fullPath)
 			if err != nil {
 				return err
@@ -220,7 +259,9 @@ func fixFormat(dir string, attachmentRegistry map[string]string) error {
 			for _, match := range re.FindAllSubmatch(data, -1) {
 				_, err := url.ParseRequestURI(string(match[2][:]))
 				if err != nil {
-					aPath := path.Clean(fmt.Sprintf("%s/%s", path.Dir(fullPath), string(match[2][:])))
+					aPath := path.Clean(
+						fmt.Sprintf("%s/%s", path.Dir(fullPath), string(match[2][:])),
+					)
 					if _, ok := attachmentRegistry[aPath]; ok {
 						link := fmt.Sprintf("![%s](%s)", match[1], attachmentRegistry[aPath])
 						data = bytes.Replace(data, match[0], []byte(link), 1)
@@ -228,17 +269,17 @@ func fixFormat(dir string, attachmentRegistry map[string]string) error {
 				}
 			}
 
-			err = ioutil.WriteFile(fullPath, data, 0644)
+			err = os.WriteFile(fullPath, data, 0644)
 
 			// format split line
-			b, err := ioutil.ReadFile(fullPath) // just pass the file name
+			b, err := os.ReadFile(fullPath) // just pass the file name
 			if err != nil {
 				return err
 			}
 
 			str := string(b)
 			str = strings.Replace(str, "\n---\n", "\n\n---\n", 1)
-			err = ioutil.WriteFile(fullPath, []byte(str), 0644)
+			err = os.WriteFile(fullPath, []byte(str), 0644)
 		}
 		return nil
 	})
@@ -334,14 +375,14 @@ func addMissingId(dir string, id int64) error {
 }
 
 func addMeta(path string, option string, value string) error {
-	b, err := ioutil.ReadFile(path) // just pass the file name
+	b, err := os.ReadFile(path) // just pass the file name
 	if err != nil {
 		return err
 	}
 
 	str := string(b)
 	str = strings.Replace(str, "\n\n---\n", fmt.Sprintf("\n* %s: %s\n\n---\n", option, value), 1)
-	err = ioutil.WriteFile(path, []byte(str), 0644)
+	err = os.WriteFile(path, []byte(str), 0644)
 
 	return err
 }
@@ -384,17 +425,18 @@ func getMaxId(dir string) (int64, error) {
 	return max, nil
 }
 
-func readPath(dirPath string, category []string, meta []string) (Tree, error) {
+func readPath(dirPath string, category []string, meta []string, config Config) (Tree, error) {
-	var clearDir = strings.TrimPrefix(dirPath, "./")
+	clearDir := strings.TrimPrefix(dirPath, "./")
 
 	tree := Tree{}
 	tree.Path = clearDir
 
-	files, err := ioutil.ReadDir(dirPath)
+	files, err := os.ReadDir(dirPath)
 	if err != nil {
 		return tree, err
 	}
 
+FILE_LOOP:
 	for _, file := range files {
 		if strings.HasPrefix(file.Name(), ".") {
 			continue
@@ -402,19 +444,29 @@ func readPath(dirPath string, category []string, meta []string) (Tree, error) {
 
 		fullPath := path.Join(dirPath, file.Name())
 		if file.IsDir() {
-			if strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, AttachmentDirName)) {
+			if strings.HasPrefix(
+				fullPath,
+				fmt.Sprintf("%s/%s", clearDir, config.AttachmentDirName),
+			) {
 				continue
 			}
 
-			nextDir, err := readPath(fullPath, append(category, file.Name()), meta)
+			nextDir, err := readPath(fullPath, append(category, file.Name()), meta, config)
 			if err != nil {
 				return tree, err
 			}
 			tree.Dirs = append(tree.Dirs, nextDir)
 			continue
+		} else {
+			for _, pattern := range config.Excludes {
+				matched, err := regexp.Match(pattern, []byte(fullPath))
+				if err == nil && matched {
+					continue FILE_LOOP
+				}
+			}
 		}
 
-		_, err := ioutil.ReadFile(fullPath)
+		_, err := os.ReadFile(fullPath)
 		if err != nil {
 			return tree, err
 		}
@@ -428,7 +480,7 @@ func readPath(dirPath string, category []string, meta []string) (Tree, error) {
 	return tree, nil
 }
 
-func readFile(file fs.FileInfo, fullPath string, category []string, meta []string) (File, error) {
+func readFile(file fs.DirEntry, fullPath string, category []string, meta []string) (File, error) {
 	f := File{
 		Name:     file.Name(),
 		FullPath: fullPath,

tree_test.go

@@ -1,14 +1,14 @@
 package zord_tree
 
 import (
-	cp "github.com/otiai10/copy"
-	"io/ioutil"
 	"os"
 	"strings"
 	"testing"
+
+	cp "github.com/otiai10/copy"
 )
 
-func prepare(t *testing.T) {
+func prepare(t *testing.T) Config {
 	err := os.RemoveAll("./testdata/sunny1")
 	if err != nil {
 	}
@@ -16,24 +16,40 @@ func prepare(t *testing.T) {
 	if err != nil {
 		t.Fatal("Couldn't prepare data for testing")
 	}
-	err = PopulateTree("./testdata/sunny1", []string{}, nil)
+
+	config := NewConfig(
+		[]string{".md"},
+		"__a",
+		make(map[string]func() string),
+		[]string{"exclude_[0-9]+.bin"},
+	)
+
+	err = PopulateTree("./testdata/sunny1", []string{}, config)
 	if err != nil {
 		t.Fatal("Population ended in err:", err)
 	}
 
+	return config
 }
 
 func TestFromNotExistingDirectory(t *testing.T) {
-	_, err := BuildTree("./testing/i_dont_exist", []string{})
+	config := NewConfig(
+		[]string{".md"},
+		"__a",
+		make(map[string]func() string),
+		[]string{},
+	)
+
+	_, err := BuildTree("./testing/i_dont_exist", []string{}, config)
 	if err == nil {
 		t.Error("Not existing directory should return error, got NIL.")
 	}
 }
 
 func TestSunny(t *testing.T) {
-	prepare(t)
+	config := prepare(t)
 
-	tree, err := BuildTree("./testdata/sunny1", []string{})
+	tree, err := BuildTree("./testdata/sunny1", []string{}, config)
 	if err != nil {
 		t.Errorf("Got error: %v", err)
 	}
@@ -60,7 +76,7 @@ func TestSunny(t *testing.T) {
 func TestFixFormat(t *testing.T) {
 	prepare(t)
 
-	b, err := ioutil.ReadFile("./testdata/sunny1/file1.md")
+	b, err := os.ReadFile("./testdata/sunny1/file1.md")
 	if err != nil {
 		t.Fatal("No destination file 'file1.md'.")
 	}
@@ -75,11 +91,10 @@ func TestFixFormat(t *testing.T) {
 }
 
 func TestMeta(t *testing.T) {
-	prepare(t)
+	config := prepare(t)
 
 	meta := []string{"option1", "option2"}
-	tree, err := BuildTree("./testdata/sunny1", meta)
+	tree, err := BuildTree("./testdata/sunny1", meta, config)
-
 	if err != nil {
 		t.Errorf("Got error: %v", err)
 	}
@@ -96,7 +111,7 @@ func TestMeta(t *testing.T) {
 func TestId(t *testing.T) {
 	prepare(t)
 
-	b, err := ioutil.ReadFile("./testdata/sunny1/file1.md")
+	b, err := os.ReadFile("./testdata/sunny1/file1.md")
 	if err != nil {
 		t.Fatal("No destination file 'file1.md'.")
 	}
@@ -105,7 +120,7 @@ func TestId(t *testing.T) {
 		t.Fatal("No id in file 'file1.md'.")
 	}
 
-	b, err = ioutil.ReadFile("./testdata/sunny1/subcategory/file3.md")
+	b, err = os.ReadFile("./testdata/sunny1/subcategory/file3.md")
 	if err != nil {
 		t.Fatal("No destination file 'substring/file3.md'.")
 	}
@@ -116,18 +131,21 @@ func TestId(t *testing.T) {
 }
 
 func TestMissingOptions(t *testing.T) {
-	prepare(t)
+	config := prepare(t)
 
 	customMeta := make(map[string]func() string)
 	customMeta["option2"] = func() string {
 		return "customDefaultValue"
 	}
-	err := PopulateTree("./testdata/sunny1", []string{"option1", "option2"}, customMeta)
+
+	config.CustomMeta = customMeta
+
+	err := PopulateTree("./testdata/sunny1", []string{"option1", "option2"}, config)
 	if err != nil {
 		t.Fatal("Population ended in err:", err)
 	}
 
-	b, err := ioutil.ReadFile("./testdata/sunny1/file1.md")
+	b, err := os.ReadFile("./testdata/sunny1/file1.md")
 	if err != nil {
 		t.Fatal("No destination file 'file1.md'.")
 	}
@@ -139,7 +157,7 @@ func TestMissingOptions(t *testing.T) {
 		t.Fatal("'option2' has not been added to file 'file1.md'.")
 	}
 
-	b, err = ioutil.ReadFile("./testdata/sunny1/file2.md")
+	b, err = os.ReadFile("./testdata/sunny1/file2.md")
 	if err != nil {
 		t.Fatal("No destination file 'file2.md'.")
 	}
@@ -151,7 +169,7 @@ func TestMissingOptions(t *testing.T) {
 		t.Fatal("'option2' has not been added to file 'file2.md'.")
 	}
 
-	b, err = ioutil.ReadFile("./testdata/sunny1/subcategory/file3.md")
+	b, err = os.ReadFile("./testdata/sunny1/subcategory/file3.md")
 	if err != nil {
 		t.Fatal("No destination file 'substring/file3.md'.")
 	}
@@ -165,10 +183,9 @@ func TestMissingOptions(t *testing.T) {
 }
 
 func TestReadingFileContent(t *testing.T) {
-	prepare(t)
+	config := prepare(t)
-
-	tree, err := BuildTree("./testdata/sunny1", []string{})
 
+	tree, err := BuildTree("./testdata/sunny1", []string{}, config)
 	if err != nil {
 		t.Errorf("Got error: %v", err)
 	}
@@ -184,9 +201,9 @@ func TestReadingFileContent(t *testing.T) {
 }
 
 func TestGetSlice(t *testing.T) {
-	prepare(t)
+	config := prepare(t)
 
-	tree, err := BuildTree("./testdata/sunny1", []string{})
+	tree, err := BuildTree("./testdata/sunny1", []string{}, config)
 	if err != nil {
 		t.Errorf("Got error: %v", err)
 	}
@@ -201,9 +218,9 @@ func TestGetSlice(t *testing.T) {
 }
 
 func TestGetFileById(t *testing.T) {
-	prepare(t)
+	config := prepare(t)
 
-	tree, err := BuildTree("./testdata/sunny1", []string{})
+	tree, err := BuildTree("./testdata/sunny1", []string{}, config)
 	if err != nil {
 		t.Errorf("Got error: %v", err)
 	}
@@ -216,3 +233,12 @@ func TestGetFileById(t *testing.T) {
 		t.Errorf("Got frong file: %s", file.Name)
 	}
 }
+
+func TestExcludes(t *testing.T) {
+	prepare(t)
+
+	_, err := os.ReadFile("./testdata/sunny1/exclude_123.bin")
+	if err != nil {
+		t.Fatal("Excludded file has been moved")
+	}
+}