From e361e38b1e15953cb45c586e511d5992a2939688 Mon Sep 17 00:00:00 2001 From: Arnas Udovic Date: Tue, 9 Jan 2024 09:43:24 +0200 Subject: [PATCH] added excludes, main Config object --- README.md | 4 +- go.mod | 2 +- testdata/sunny/exclude_123.bin | 0 tree.go | 120 +++++++++++++++++++++++---------- tree_test.go | 76 ++++++++++++++------- 5 files changed, 141 insertions(+), 61 deletions(-) create mode 100644 testdata/sunny/exclude_123.bin diff --git a/README.md b/README.md index 814c222..ad43441 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ content. For now it supports markdown format. Process should go in two steps: * populate existing `source` directory with formatting files, add missing `id` and meta data -* separate images and other data files to `__a` root directory +* separate images and other data files to ex. `__a` root directory * build `Tree` to operate on object for your app * read file content without meta data by path @@ -64,6 +64,8 @@ Hear goes content. It can be written in html, markdown or whatever what can be p There are two main commands: `PopulateTree` to prepare source (format, add metadata and add Id) and separate images and other data files and `BuildTree` to get object of tree. +Also both commands use `Config` object that can be prepared by `NewConfig` command. With it can be configurable readable formats, attachment directory, custom meta and excludes (not movable files by regexp). + `Tree` object has methods: `FileById` to get `File` by Id, `Slice` to get sub-tree of given path and `Filter` to filter tree by filter. Filter contains array of meta key and searching value. `tag` key is searched as equal and other meta values of keys can contain part. diff --git a/go.mod b/go.mod index 8e0f632..43213d7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module g.arns.lt/zordsdavini/zord-tree -go 1.18 +go 1.21 require ( g.arns.lt/zordsdavini/abcex v1.0.0 diff --git a/testdata/sunny/exclude_123.bin b/testdata/sunny/exclude_123.bin new file mode 100644 index 0000000..e69de29 diff --git a/tree.go b/tree.go index 4ffc65e..7f815aa 100644 --- a/tree.go +++ b/tree.go @@ -6,19 +6,26 @@ import ( "crypto/md5" "errors" "fmt" - "g.arns.lt/zordsdavini/abcex" - "golang.org/x/exp/slices" "io" "io/fs" - "io/ioutil" "net/url" "os" "path" "path/filepath" "regexp" "strings" + + "g.arns.lt/zordsdavini/abcex" + "golang.org/x/exp/slices" ) +type Config struct { + ReadableFormats []string + AttachmentDirName string + CustomMeta map[string]func() string + Excludes []string +} + type File struct { Id string Name string @@ -34,10 +41,6 @@ type Tree struct { Files []File } -var readableFormats = []string{".md"} - -const AttachmentDirName = "__a" - func (t Tree) FileById(id string) (File, error) { for _, f := range t.Files { if f.Id == id { @@ -108,20 +111,34 @@ func (t Tree) Filter(filter map[string][]string) (Tree, bool) { return filtered, found } -func BuildTree(dirPath string, meta []string) (Tree, error) { - return readPath(dirPath, []string{}, meta) +func NewConfig( + readableFormats []string, + attachmentDirName string, + customMeta map[string]func() string, + excludes []string, +) Config { + return Config{ + ReadableFormats: readableFormats, + AttachmentDirName: attachmentDirName, + CustomMeta: customMeta, + Excludes: excludes, + } } -func PopulateTree(sourcePath string, meta []string, customMeta map[string]func() string) error { +func BuildTree(dirPath string, meta []string, config Config) (Tree, error) { + return readPath(dirPath, []string{}, meta, config) +} + +func PopulateTree(sourcePath string, meta []string, config Config) error { var err error var attachmentRegistry map[string]string - attachmentRegistry, err = moveAttachments(sourcePath) + attachmentRegistry, err = moveAttachments(sourcePath, config) if err != nil { return err } - err = fixFormat(sourcePath, attachmentRegistry) + err = fixFormat(sourcePath, attachmentRegistry, config) if err != nil { return err } @@ -138,12 +155,12 @@ func PopulateTree(sourcePath string, meta []string, customMeta map[string]func() return err } - err = addMissingMeta(sourcePath, meta, customMeta) + err = addMissingMeta(sourcePath, meta, config.CustomMeta) return err } -func moveAttachments(dir string) (map[string]string, error) { +func moveAttachments(dir string, config Config) (map[string]string, error) { attachmentRegistry := make(map[string]string) err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error { @@ -151,8 +168,19 @@ func moveAttachments(dir string) (map[string]string, error) { return e } - var clearDir = strings.TrimPrefix(dir, "./") - if info.Mode().IsRegular() && !slices.Contains(readableFormats, path.Ext(fullPath)) && !strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, AttachmentDirName)) { + clearDir := strings.TrimPrefix(dir, "./") + if info.Mode().IsRegular() { + for _, pattern := range config.Excludes { + matched, err := regexp.Match(pattern, []byte(fullPath)) + if err == nil && matched { + return nil + } + } + } + + if info.Mode().IsRegular() && + !slices.Contains(config.ReadableFormats, path.Ext(fullPath)) && + !strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, config.AttachmentDirName)) { f, err := os.Open(fullPath) if err != nil { return err @@ -164,10 +192,21 @@ func moveAttachments(dir string) (map[string]string, error) { return err } - _ = os.Mkdir(fmt.Sprintf("%s/%s", dir, AttachmentDirName), 0755) + _ = os.Mkdir(fmt.Sprintf("%s/%s", dir, config.AttachmentDirName), 0755) - attachmentRegistry[fullPath] = fmt.Sprintf("%s/%x%s", AttachmentDirName, h.Sum(nil), path.Ext(fullPath)) - newPath := fmt.Sprintf("%s/%s/%x%s", dir, AttachmentDirName, h.Sum(nil), path.Ext(fullPath)) + attachmentRegistry[fullPath] = fmt.Sprintf( + "%s/%x%s", + config.AttachmentDirName, + h.Sum(nil), + path.Ext(fullPath), + ) + newPath := fmt.Sprintf( + "%s/%s/%x%s", + dir, + config.AttachmentDirName, + h.Sum(nil), + path.Ext(fullPath), + ) err = os.Rename(fullPath, newPath) if err != nil { return err @@ -180,13 +219,13 @@ func moveAttachments(dir string) (map[string]string, error) { return attachmentRegistry, err } -func fixFormat(dir string, attachmentRegistry map[string]string) error { +func fixFormat(dir string, attachmentRegistry map[string]string, config Config) error { err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error { if e != nil { return e } - if info.Mode().IsRegular() && slices.Contains(readableFormats, path.Ext(fullPath)) { + if info.Mode().IsRegular() && slices.Contains(config.ReadableFormats, path.Ext(fullPath)) { osf, err := os.Open(fullPath) if err != nil { return err @@ -220,7 +259,9 @@ func fixFormat(dir string, attachmentRegistry map[string]string) error { for _, match := range re.FindAllSubmatch(data, -1) { _, err := url.ParseRequestURI(string(match[2][:])) if err != nil { - aPath := path.Clean(fmt.Sprintf("%s/%s", path.Dir(fullPath), string(match[2][:]))) + aPath := path.Clean( + fmt.Sprintf("%s/%s", path.Dir(fullPath), string(match[2][:])), + ) if _, ok := attachmentRegistry[aPath]; ok { link := fmt.Sprintf("![%s](%s)", match[1], attachmentRegistry[aPath]) data = bytes.Replace(data, match[0], []byte(link), 1) @@ -228,17 +269,17 @@ func fixFormat(dir string, attachmentRegistry map[string]string) error { } } - err = ioutil.WriteFile(fullPath, data, 0644) + err = os.WriteFile(fullPath, data, 0644) // format split line - b, err := ioutil.ReadFile(fullPath) // just pass the file name + b, err := os.ReadFile(fullPath) // just pass the file name if err != nil { return err } str := string(b) str = strings.Replace(str, "\n---\n", "\n\n---\n", 1) - err = ioutil.WriteFile(fullPath, []byte(str), 0644) + err = os.WriteFile(fullPath, []byte(str), 0644) } return nil }) @@ -334,14 +375,14 @@ func addMissingId(dir string, id int64) error { } func addMeta(path string, option string, value string) error { - b, err := ioutil.ReadFile(path) // just pass the file name + b, err := os.ReadFile(path) // just pass the file name if err != nil { return err } str := string(b) str = strings.Replace(str, "\n\n---\n", fmt.Sprintf("\n* %s: %s\n\n---\n", option, value), 1) - err = ioutil.WriteFile(path, []byte(str), 0644) + err = os.WriteFile(path, []byte(str), 0644) return err } @@ -384,17 +425,18 @@ func getMaxId(dir string) (int64, error) { return max, nil } -func readPath(dirPath string, category []string, meta []string) (Tree, error) { - var clearDir = strings.TrimPrefix(dirPath, "./") +func readPath(dirPath string, category []string, meta []string, config Config) (Tree, error) { + clearDir := strings.TrimPrefix(dirPath, "./") tree := Tree{} tree.Path = clearDir - files, err := ioutil.ReadDir(dirPath) + files, err := os.ReadDir(dirPath) if err != nil { return tree, err } +FILE_LOOP: for _, file := range files { if strings.HasPrefix(file.Name(), ".") { continue @@ -402,19 +444,29 @@ func readPath(dirPath string, category []string, meta []string) (Tree, error) { fullPath := path.Join(dirPath, file.Name()) if file.IsDir() { - if strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, AttachmentDirName)) { + if strings.HasPrefix( + fullPath, + fmt.Sprintf("%s/%s", clearDir, config.AttachmentDirName), + ) { continue } - nextDir, err := readPath(fullPath, append(category, file.Name()), meta) + nextDir, err := readPath(fullPath, append(category, file.Name()), meta, config) if err != nil { return tree, err } tree.Dirs = append(tree.Dirs, nextDir) continue + } else { + for _, pattern := range config.Excludes { + matched, err := regexp.Match(pattern, []byte(fullPath)) + if err == nil && matched { + continue FILE_LOOP + } + } } - _, err := ioutil.ReadFile(fullPath) + _, err := os.ReadFile(fullPath) if err != nil { return tree, err } @@ -428,7 +480,7 @@ func readPath(dirPath string, category []string, meta []string) (Tree, error) { return tree, nil } -func readFile(file fs.FileInfo, fullPath string, category []string, meta []string) (File, error) { +func readFile(file fs.DirEntry, fullPath string, category []string, meta []string) (File, error) { f := File{ Name: file.Name(), FullPath: fullPath, diff --git a/tree_test.go b/tree_test.go index eca71c2..b35ece4 100644 --- a/tree_test.go +++ b/tree_test.go @@ -1,14 +1,14 @@ package zord_tree import ( - cp "github.com/otiai10/copy" - "io/ioutil" "os" "strings" "testing" + + cp "github.com/otiai10/copy" ) -func prepare(t *testing.T) { +func prepare(t *testing.T) Config { err := os.RemoveAll("./testdata/sunny1") if err != nil { } @@ -16,24 +16,40 @@ func prepare(t *testing.T) { if err != nil { t.Fatal("Couldn't prepare data for testing") } - err = PopulateTree("./testdata/sunny1", []string{}, nil) + + config := NewConfig( + []string{".md"}, + "__a", + make(map[string]func() string), + []string{"exclude_[0-9]+.bin"}, + ) + + err = PopulateTree("./testdata/sunny1", []string{}, config) if err != nil { t.Fatal("Population ended in err:", err) } + return config } func TestFromNotExistingDirectory(t *testing.T) { - _, err := BuildTree("./testing/i_dont_exist", []string{}) + config := NewConfig( + []string{".md"}, + "__a", + make(map[string]func() string), + []string{}, + ) + + _, err := BuildTree("./testing/i_dont_exist", []string{}, config) if err == nil { t.Error("Not existing directory should return error, got NIL.") } } func TestSunny(t *testing.T) { - prepare(t) + config := prepare(t) - tree, err := BuildTree("./testdata/sunny1", []string{}) + tree, err := BuildTree("./testdata/sunny1", []string{}, config) if err != nil { t.Errorf("Got error: %v", err) } @@ -60,7 +76,7 @@ func TestSunny(t *testing.T) { func TestFixFormat(t *testing.T) { prepare(t) - b, err := ioutil.ReadFile("./testdata/sunny1/file1.md") + b, err := os.ReadFile("./testdata/sunny1/file1.md") if err != nil { t.Fatal("No destination file 'file1.md'.") } @@ -75,11 +91,10 @@ func TestFixFormat(t *testing.T) { } func TestMeta(t *testing.T) { - prepare(t) + config := prepare(t) meta := []string{"option1", "option2"} - tree, err := BuildTree("./testdata/sunny1", meta) - + tree, err := BuildTree("./testdata/sunny1", meta, config) if err != nil { t.Errorf("Got error: %v", err) } @@ -96,7 +111,7 @@ func TestMeta(t *testing.T) { func TestId(t *testing.T) { prepare(t) - b, err := ioutil.ReadFile("./testdata/sunny1/file1.md") + b, err := os.ReadFile("./testdata/sunny1/file1.md") if err != nil { t.Fatal("No destination file 'file1.md'.") } @@ -105,7 +120,7 @@ func TestId(t *testing.T) { t.Fatal("No id in file 'file1.md'.") } - b, err = ioutil.ReadFile("./testdata/sunny1/subcategory/file3.md") + b, err = os.ReadFile("./testdata/sunny1/subcategory/file3.md") if err != nil { t.Fatal("No destination file 'substring/file3.md'.") } @@ -116,18 +131,21 @@ func TestId(t *testing.T) { } func TestMissingOptions(t *testing.T) { - prepare(t) + config := prepare(t) customMeta := make(map[string]func() string) customMeta["option2"] = func() string { return "customDefaultValue" } - err := PopulateTree("./testdata/sunny1", []string{"option1", "option2"}, customMeta) + + config.CustomMeta = customMeta + + err := PopulateTree("./testdata/sunny1", []string{"option1", "option2"}, config) if err != nil { t.Fatal("Population ended in err:", err) } - b, err := ioutil.ReadFile("./testdata/sunny1/file1.md") + b, err := os.ReadFile("./testdata/sunny1/file1.md") if err != nil { t.Fatal("No destination file 'file1.md'.") } @@ -139,7 +157,7 @@ func TestMissingOptions(t *testing.T) { t.Fatal("'option2' has not been added to file 'file1.md'.") } - b, err = ioutil.ReadFile("./testdata/sunny1/file2.md") + b, err = os.ReadFile("./testdata/sunny1/file2.md") if err != nil { t.Fatal("No destination file 'file2.md'.") } @@ -151,7 +169,7 @@ func TestMissingOptions(t *testing.T) { t.Fatal("'option2' has not been added to file 'file2.md'.") } - b, err = ioutil.ReadFile("./testdata/sunny1/subcategory/file3.md") + b, err = os.ReadFile("./testdata/sunny1/subcategory/file3.md") if err != nil { t.Fatal("No destination file 'substring/file3.md'.") } @@ -165,10 +183,9 @@ func TestMissingOptions(t *testing.T) { } func TestReadingFileContent(t *testing.T) { - prepare(t) - - tree, err := BuildTree("./testdata/sunny1", []string{}) + config := prepare(t) + tree, err := BuildTree("./testdata/sunny1", []string{}, config) if err != nil { t.Errorf("Got error: %v", err) } @@ -184,9 +201,9 @@ func TestReadingFileContent(t *testing.T) { } func TestGetSlice(t *testing.T) { - prepare(t) + config := prepare(t) - tree, err := BuildTree("./testdata/sunny1", []string{}) + tree, err := BuildTree("./testdata/sunny1", []string{}, config) if err != nil { t.Errorf("Got error: %v", err) } @@ -201,9 +218,9 @@ func TestGetSlice(t *testing.T) { } func TestGetFileById(t *testing.T) { - prepare(t) + config := prepare(t) - tree, err := BuildTree("./testdata/sunny1", []string{}) + tree, err := BuildTree("./testdata/sunny1", []string{}, config) if err != nil { t.Errorf("Got error: %v", err) } @@ -216,3 +233,12 @@ func TestGetFileById(t *testing.T) { t.Errorf("Got frong file: %s", file.Name) } } + +func TestExcludes(t *testing.T) { + prepare(t) + + _, err := os.ReadFile("./testdata/sunny1/exclude_123.bin") + if err != nil { + t.Fatal("Excludded file has been moved") + } +}