zordsdavini/zord-tree
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

added excludes, main Config object

Browse source
This commit is contained in:
Arnas Udovic 2024-01-09 09:43:24 +02:00
parent 7034e2aad8
commit e361e38b1e
5 changed files with 141 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -8,7 +8,7 @@ content. For now it supports markdown format.
Process should go in two steps:
* populate existing `source` directory with formatting files, add missing `id` and meta data
* separate images and other data files to `__a` root directory
* separate images and other data files to ex. `__a` root directory
* build `Tree` to operate on object for your app
* read file content without meta data by path
@ -64,6 +64,8 @@ Hear goes content. It can be written in html, markdown or whatever what can be p
There are two main commands: `PopulateTree` to prepare source (format, add metadata and add Id) and separate images and other data files and `BuildTree` to get object of tree.
Also both commands use `Config` object that can be prepared by `NewConfig` command. With it can be configurable readable formats, attachment directory, custom meta and excludes (not movable files by regexp).
`Tree` object has methods: `FileById` to get `File` by Id, `Slice` to get sub-tree of given path and `Filter` to filter tree by filter.
Filter contains array of meta key and searching value. `tag` key is searched as equal and other meta values of keys can contain part.

2
go.mod
View file

@ -1,6 +1,6 @@
module g.arns.lt/zordsdavini/zord-tree
go 1.18
go 1.21
require (
g.arns.lt/zordsdavini/abcex v1.0.0

0
testdata/sunny/exclude_123.bin vendored Normal file
View file

120
tree.go
View file

@ -6,19 +6,26 @@ import (
"crypto/md5"
"errors"
"fmt"
"g.arns.lt/zordsdavini/abcex"
"golang.org/x/exp/slices"
"io"
"io/fs"
"io/ioutil"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"g.arns.lt/zordsdavini/abcex"
"golang.org/x/exp/slices"
)
type Config struct {
ReadableFormats []string
AttachmentDirName string
CustomMeta map[string]func() string
Excludes []string
}
type File struct {
Id string
Name string
@ -34,10 +41,6 @@ type Tree struct {
Files []File
}
var readableFormats = []string{".md"}
const AttachmentDirName = "__a"
func (t Tree) FileById(id string) (File, error) {
for _, f := range t.Files {
if f.Id == id {
@ -108,20 +111,34 @@ func (t Tree) Filter(filter map[string][]string) (Tree, bool) {
return filtered, found
}
func BuildTree(dirPath string, meta []string) (Tree, error) {
return readPath(dirPath, []string{}, meta)
func NewConfig(
readableFormats []string,
attachmentDirName string,
customMeta map[string]func() string,
excludes []string,
) Config {
return Config{
ReadableFormats: readableFormats,
AttachmentDirName: attachmentDirName,
CustomMeta: customMeta,
Excludes: excludes,
}
}
func PopulateTree(sourcePath string, meta []string, customMeta map[string]func() string) error {
func BuildTree(dirPath string, meta []string, config Config) (Tree, error) {
return readPath(dirPath, []string{}, meta, config)
}
func PopulateTree(sourcePath string, meta []string, config Config) error {
var err error
var attachmentRegistry map[string]string
attachmentRegistry, err = moveAttachments(sourcePath)
attachmentRegistry, err = moveAttachments(sourcePath, config)
if err != nil {
return err
}
err = fixFormat(sourcePath, attachmentRegistry)
err = fixFormat(sourcePath, attachmentRegistry, config)
if err != nil {
return err
}
@ -138,12 +155,12 @@ func PopulateTree(sourcePath string, meta []string, customMeta map[string]func()
return err
}
err = addMissingMeta(sourcePath, meta, customMeta)
err = addMissingMeta(sourcePath, meta, config.CustomMeta)
return err
}
func moveAttachments(dir string) (map[string]string, error) {
func moveAttachments(dir string, config Config) (map[string]string, error) {
attachmentRegistry := make(map[string]string)
err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error {
@ -151,8 +168,19 @@ func moveAttachments(dir string) (map[string]string, error) {
return e
}
var clearDir = strings.TrimPrefix(dir, "./")
if info.Mode().IsRegular() && !slices.Contains(readableFormats, path.Ext(fullPath)) && !strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, AttachmentDirName)) {
clearDir := strings.TrimPrefix(dir, "./")
if info.Mode().IsRegular() {
for _, pattern := range config.Excludes {
matched, err := regexp.Match(pattern, []byte(fullPath))
if err == nil && matched {
return nil
}
}
}
if info.Mode().IsRegular() &&
!slices.Contains(config.ReadableFormats, path.Ext(fullPath)) &&
!strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, config.AttachmentDirName)) {
f, err := os.Open(fullPath)
if err != nil {
return err
@ -164,10 +192,21 @@ func moveAttachments(dir string) (map[string]string, error) {
return err
}
_ = os.Mkdir(fmt.Sprintf("%s/%s", dir, AttachmentDirName), 0755)
_ = os.Mkdir(fmt.Sprintf("%s/%s", dir, config.AttachmentDirName), 0755)
attachmentRegistry[fullPath] = fmt.Sprintf("%s/%x%s", AttachmentDirName, h.Sum(nil), path.Ext(fullPath))
newPath := fmt.Sprintf("%s/%s/%x%s", dir, AttachmentDirName, h.Sum(nil), path.Ext(fullPath))
attachmentRegistry[fullPath] = fmt.Sprintf(
"%s/%x%s",
config.AttachmentDirName,
h.Sum(nil),
path.Ext(fullPath),
)
newPath := fmt.Sprintf(
"%s/%s/%x%s",
dir,
config.AttachmentDirName,
h.Sum(nil),
path.Ext(fullPath),
)
err = os.Rename(fullPath, newPath)
if err != nil {
return err
@ -180,13 +219,13 @@ func moveAttachments(dir string) (map[string]string, error) {
return attachmentRegistry, err
}
func fixFormat(dir string, attachmentRegistry map[string]string) error {
func fixFormat(dir string, attachmentRegistry map[string]string, config Config) error {
err := filepath.Walk(dir, func(fullPath string, info os.FileInfo, e error) error {
if e != nil {
return e
}
if info.Mode().IsRegular() && slices.Contains(readableFormats, path.Ext(fullPath)) {
if info.Mode().IsRegular() && slices.Contains(config.ReadableFormats, path.Ext(fullPath)) {
osf, err := os.Open(fullPath)
if err != nil {
return err
@ -220,7 +259,9 @@ func fixFormat(dir string, attachmentRegistry map[string]string) error {
for _, match := range re.FindAllSubmatch(data, -1) {
_, err := url.ParseRequestURI(string(match[2][:]))
if err != nil {
aPath := path.Clean(fmt.Sprintf("%s/%s", path.Dir(fullPath), string(match[2][:])))
aPath := path.Clean(
fmt.Sprintf("%s/%s", path.Dir(fullPath), string(match[2][:])),
)
if _, ok := attachmentRegistry[aPath]; ok {
link := fmt.Sprintf("![%s](%s)", match[1], attachmentRegistry[aPath])
data = bytes.Replace(data, match[0], []byte(link), 1)
@ -228,17 +269,17 @@ func fixFormat(dir string, attachmentRegistry map[string]string) error {
}
}
err = ioutil.WriteFile(fullPath, data, 0644)
err = os.WriteFile(fullPath, data, 0644)
// format split line
b, err := ioutil.ReadFile(fullPath) // just pass the file name
b, err := os.ReadFile(fullPath) // just pass the file name
if err != nil {
return err
}
str := string(b)
str = strings.Replace(str, "\n---\n", "\n\n---\n", 1)
err = ioutil.WriteFile(fullPath, []byte(str), 0644)
err = os.WriteFile(fullPath, []byte(str), 0644)
}
return nil
})
@ -334,14 +375,14 @@ func addMissingId(dir string, id int64) error {
}
func addMeta(path string, option string, value string) error {
b, err := ioutil.ReadFile(path) // just pass the file name
b, err := os.ReadFile(path) // just pass the file name
if err != nil {
return err
}
str := string(b)
str = strings.Replace(str, "\n\n---\n", fmt.Sprintf("\n* %s: %s\n\n---\n", option, value), 1)
err = ioutil.WriteFile(path, []byte(str), 0644)
err = os.WriteFile(path, []byte(str), 0644)
return err
}
@ -384,17 +425,18 @@ func getMaxId(dir string) (int64, error) {
return max, nil
}
func readPath(dirPath string, category []string, meta []string) (Tree, error) {
var clearDir = strings.TrimPrefix(dirPath, "./")
func readPath(dirPath string, category []string, meta []string, config Config) (Tree, error) {
clearDir := strings.TrimPrefix(dirPath, "./")
tree := Tree{}
tree.Path = clearDir
files, err := ioutil.ReadDir(dirPath)
files, err := os.ReadDir(dirPath)
if err != nil {
return tree, err
}
FILE_LOOP:
for _, file := range files {
if strings.HasPrefix(file.Name(), ".") {
continue
@ -402,19 +444,29 @@ func readPath(dirPath string, category []string, meta []string) (Tree, error) {
fullPath := path.Join(dirPath, file.Name())
if file.IsDir() {
if strings.HasPrefix(fullPath, fmt.Sprintf("%s/%s", clearDir, AttachmentDirName)) {
if strings.HasPrefix(
fullPath,
fmt.Sprintf("%s/%s", clearDir, config.AttachmentDirName),
) {
continue
}
nextDir, err := readPath(fullPath, append(category, file.Name()), meta)
nextDir, err := readPath(fullPath, append(category, file.Name()), meta, config)
if err != nil {
return tree, err
}
tree.Dirs = append(tree.Dirs, nextDir)
continue
} else {
for _, pattern := range config.Excludes {
matched, err := regexp.Match(pattern, []byte(fullPath))
if err == nil && matched {
continue FILE_LOOP
}
}
}
_, err := ioutil.ReadFile(fullPath)
_, err := os.ReadFile(fullPath)
if err != nil {
return tree, err
}
@ -428,7 +480,7 @@ func readPath(dirPath string, category []string, meta []string) (Tree, error) {
return tree, nil
}
func readFile(file fs.FileInfo, fullPath string, category []string, meta []string) (File, error) {
func readFile(file fs.DirEntry, fullPath string, category []string, meta []string) (File, error) {
f := File{
Name: file.Name(),
FullPath: fullPath,

76
tree_test.go
View file

@ -1,14 +1,14 @@
package zord_tree
import (
cp "github.com/otiai10/copy"
"io/ioutil"
"os"
"strings"
"testing"
cp "github.com/otiai10/copy"
)
func prepare(t *testing.T) {
func prepare(t *testing.T) Config {
err := os.RemoveAll("./testdata/sunny1")
if err != nil {
}
@ -16,24 +16,40 @@ func prepare(t *testing.T) {
if err != nil {
t.Fatal("Couldn't prepare data for testing")
}
err = PopulateTree("./testdata/sunny1", []string{}, nil)
config := NewConfig(
[]string{".md"},
"__a",
make(map[string]func() string),
[]string{"exclude_[0-9]+.bin"},
)
err = PopulateTree("./testdata/sunny1", []string{}, config)
if err != nil {
t.Fatal("Population ended in err:", err)
}
return config
}
func TestFromNotExistingDirectory(t *testing.T) {
_, err := BuildTree("./testing/i_dont_exist", []string{})
config := NewConfig(
[]string{".md"},
"__a",
make(map[string]func() string),
[]string{},
)
_, err := BuildTree("./testing/i_dont_exist", []string{}, config)
if err == nil {
t.Error("Not existing directory should return error, got NIL.")
}
}
func TestSunny(t *testing.T) {
prepare(t)
config := prepare(t)
tree, err := BuildTree("./testdata/sunny1", []string{})
tree, err := BuildTree("./testdata/sunny1", []string{}, config)
if err != nil {
t.Errorf("Got error: %v", err)
}
@ -60,7 +76,7 @@ func TestSunny(t *testing.T) {
func TestFixFormat(t *testing.T) {
prepare(t)
b, err := ioutil.ReadFile("./testdata/sunny1/file1.md")
b, err := os.ReadFile("./testdata/sunny1/file1.md")
if err != nil {
t.Fatal("No destination file 'file1.md'.")
}
@ -75,11 +91,10 @@ func TestFixFormat(t *testing.T) {
}
func TestMeta(t *testing.T) {
prepare(t)
config := prepare(t)
meta := []string{"option1", "option2"}
tree, err := BuildTree("./testdata/sunny1", meta)
tree, err := BuildTree("./testdata/sunny1", meta, config)
if err != nil {
t.Errorf("Got error: %v", err)
}
@ -96,7 +111,7 @@ func TestMeta(t *testing.T) {
func TestId(t *testing.T) {
prepare(t)
b, err := ioutil.ReadFile("./testdata/sunny1/file1.md")
b, err := os.ReadFile("./testdata/sunny1/file1.md")
if err != nil {
t.Fatal("No destination file 'file1.md'.")
}
@ -105,7 +120,7 @@ func TestId(t *testing.T) {
t.Fatal("No id in file 'file1.md'.")
}
b, err = ioutil.ReadFile("./testdata/sunny1/subcategory/file3.md")
b, err = os.ReadFile("./testdata/sunny1/subcategory/file3.md")
if err != nil {
t.Fatal("No destination file 'substring/file3.md'.")
}
@ -116,18 +131,21 @@ func TestId(t *testing.T) {
}
func TestMissingOptions(t *testing.T) {
prepare(t)
config := prepare(t)
customMeta := make(map[string]func() string)
customMeta["option2"] = func() string {
return "customDefaultValue"
}
err := PopulateTree("./testdata/sunny1", []string{"option1", "option2"}, customMeta)
config.CustomMeta = customMeta
err := PopulateTree("./testdata/sunny1", []string{"option1", "option2"}, config)
if err != nil {
t.Fatal("Population ended in err:", err)
}
b, err := ioutil.ReadFile("./testdata/sunny1/file1.md")
b, err := os.ReadFile("./testdata/sunny1/file1.md")
if err != nil {
t.Fatal("No destination file 'file1.md'.")
}
@ -139,7 +157,7 @@ func TestMissingOptions(t *testing.T) {
t.Fatal("'option2' has not been added to file 'file1.md'.")
}
b, err = ioutil.ReadFile("./testdata/sunny1/file2.md")
b, err = os.ReadFile("./testdata/sunny1/file2.md")
if err != nil {
t.Fatal("No destination file 'file2.md'.")
}
@ -151,7 +169,7 @@ func TestMissingOptions(t *testing.T) {
t.Fatal("'option2' has not been added to file 'file2.md'.")
}
b, err = ioutil.ReadFile("./testdata/sunny1/subcategory/file3.md")
b, err = os.ReadFile("./testdata/sunny1/subcategory/file3.md")
if err != nil {
t.Fatal("No destination file 'substring/file3.md'.")
}
@ -165,10 +183,9 @@ func TestMissingOptions(t *testing.T) {
}
func TestReadingFileContent(t *testing.T) {
prepare(t)
tree, err := BuildTree("./testdata/sunny1", []string{})
config := prepare(t)
tree, err := BuildTree("./testdata/sunny1", []string{}, config)
if err != nil {
t.Errorf("Got error: %v", err)
}
@ -184,9 +201,9 @@ func TestReadingFileContent(t *testing.T) {
}
func TestGetSlice(t *testing.T) {
prepare(t)
config := prepare(t)
tree, err := BuildTree("./testdata/sunny1", []string{})
tree, err := BuildTree("./testdata/sunny1", []string{}, config)
if err != nil {
t.Errorf("Got error: %v", err)
}
@ -201,9 +218,9 @@ func TestGetSlice(t *testing.T) {
}
func TestGetFileById(t *testing.T) {
prepare(t)
config := prepare(t)
tree, err := BuildTree("./testdata/sunny1", []string{})
tree, err := BuildTree("./testdata/sunny1", []string{}, config)
if err != nil {
t.Errorf("Got error: %v", err)
}
@ -216,3 +233,12 @@ func TestGetFileById(t *testing.T) {
t.Errorf("Got frong file: %s", file.Name)
}
}
func TestExcludes(t *testing.T) {
prepare(t)
_, err := os.ReadFile("./testdata/sunny1/exclude_123.bin")
if err != nil {
t.Fatal("Excludded file has been moved")
}
}