Skip to content

Commit

Permalink
split, atom mode for init
Browse files Browse the repository at this point in the history
  • Loading branch information
laktak committed Dec 31, 2024
1 parent 50807f7 commit 1577cc2
Show file tree
Hide file tree
Showing 11 changed files with 341 additions and 221 deletions.
21 changes: 9 additions & 12 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,6 @@ intact over time, especially during transfers and backups.
Flags:
-h, --help Show context-sensitive help.
--db use a index database instead of index files
-m, --[no-]show-missing show missing files/directories
-d, --[no-]include-dot include dot files
-S, --[no-]skip-symlinks do not follow symlinks
Expand All @@ -117,13 +116,10 @@ Commands:
chkbit will verify files in readonly mode
update <paths> ... [flags]
add and update indices
add and update indices (see flags with -h)
init-db <path> [flags]
initialize a new index database at the given path for use with --db
export-db <path> [flags]
export a database to a json for archiving
init <mode> <path> [flags]
initialize a new index at the given path
show-ignored-only <paths> ... [flags]
only show ignored files
Expand Down Expand Up @@ -152,15 +148,16 @@ $ chkbit tips
- lines starting with '/' are only applied to the current directory
Status codes:
PNC: exception/panic, unable to continue
DMG: error, data damage detected
EIX: error, index damaged
ERX: error, index damaged
old: warning, file replaced by an older version
new: new file
upd: file updated
ok : check ok
del: file/directory removed
new: new file
ok : checked and ok (verbose)
del: file/directory removed (-m)
ign: ignored (see .chkbitignore)
EXC: exception/panic
msg: message
Configuration file (json):
- location /home/spark/.config/chkbit/config.json
Expand Down
11 changes: 6 additions & 5 deletions cmd/chkbit/help.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,16 @@ var helpTips = `
- lines starting with '/' are only applied to the current directory
Status codes:
PNC: exception/panic, unable to continue
DMG: error, data damage detected
EIX: error, index damaged
ERX: error, index damaged
old: warning, file replaced by an older version
new: new file
upd: file updated
ok : check ok
del: file/directory removed
new: new file
ok : checked and ok (verbose)
del: file/directory removed (-m)
ign: ignored (see .chkbitignore)
EXC: exception/panic
msg: message
Configuration file (json):
- location <config-file>
Expand Down
41 changes: 25 additions & 16 deletions cmd/chkbit/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,13 @@ type CLI struct {
Paths []string `arg:"" name:"paths" help:"directories to update"`
AddOnly bool `short:"a" help:"only add new and modified files, do not check existing (quicker)"`
Force bool `help:"force update of damaged items (advanced usage only)"`
} `cmd:"" help:"add and update indices"`
} `cmd:"" help:"add and update indices (see flags with -h)"`

InitDb struct {
Path string `arg:"" help:"directory for the database"`
Force bool `help:"force init if a database already exists"`
} `cmd:"" help:"initialize a new index database at the given path for use with --db"`
Init struct {
Mode string `arg:"" enum:"split,atom" help:"the mode defines if you wish to store on index per directory (split) or one index in the given path (atom)"`
Path string `arg:"" help:"directory for the store root"`
Force bool `help:"force init if a store already exists"`
} `cmd:"" help:"initialize a new index at the given path"`

ShowIgnoredOnly struct {
Paths []string `arg:"" name:"paths" help:"directories to list"`
Expand All @@ -77,7 +78,6 @@ type CLI struct {
Version struct {
} `cmd:"" help:"show version information"`

Db bool `help:"use a index database instead of index files"`
ShowMissing bool `short:"m" help:"show missing files/directories" negatable:""`
IncludeDot bool `short:"d" help:"include dot files" negatable:""`
SkipSymlinks bool `short:"S" help:"do not follow symlinks" negatable:""`
Expand Down Expand Up @@ -257,15 +257,20 @@ func (m *Main) process(cmd Command, cli CLI) (bool, error) {
m.context.SkipSubdirectories = cli.NoRecurse
m.context.TrackDirectories = !cli.NoDirInIndex

if cli.Db {
var root string
root, pathList, err = m.context.UseStoreDb(pathList)
st, root, err := chkbit.LocateStore(pathList[0], chkbit.StoreTypeAny, m.context.IndexFilename)
if err != nil {
return false, err
}

if st == chkbit.StoreTypeAtom {
pathList, err = m.context.UseAtomStore(root, pathList)
if err == nil {
// pathList is relative to root
err = os.Chdir(root)
m.logInfo("", "Using store-db in "+root)
}
if err != nil {
if err = os.Chdir(root); err != nil {
return false, err
}
m.logInfo("", "Using atom-store in "+root)
} else {
return false, err
}
}
Expand Down Expand Up @@ -394,9 +399,13 @@ func (m *Main) run() int {
cmd = Update
case "show-ignored-only <paths>":
cmd = Show
case "init-db <path>":
m.logInfo("", "chkbit init-db "+cli.InitDb.Path)
if err := chkbit.InitializeIndexDb(cli.InitDb.Path, cli.IndexName, cli.InitDb.Force); err != nil {
case "init <mode> <path>":
m.logInfo("", fmt.Sprintf("chkbit init %s %s", cli.Init.Mode, cli.Init.Path))
st := chkbit.StoreTypeSplit
if cli.Init.Mode == "atom" {
st = chkbit.StoreTypeAtom
}
if err := chkbit.InitializeStore(st, cli.Init.Path, cli.IndexName, cli.Init.Force); err != nil {
m.logError(err.Error())
return 1
}
Expand Down
44 changes: 19 additions & 25 deletions context.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ func (context *Context) endWork() {
}

func (context *Context) isChkbitFile(name string) bool {
// any file with the index prefix is ignored (to allow for .bak and db files)
// any file with the index prefix is ignored (to allow for .bak and -db files)
return strings.HasPrefix(name, context.IndexFilename) || name == context.IgnoreFilename
}

Expand Down Expand Up @@ -152,7 +152,8 @@ func (context *Context) Process(pathList []string) {
if updated, err := context.store.Finish(); err != nil {
context.logErr("index", err)
} else if updated {
context.log(StatusInfo, "The index db was updated")
// todo
// context.log(StatusInfo, "The index store was updated")
}
context.LogQueue <- nil
}
Expand Down Expand Up @@ -209,34 +210,27 @@ func (context *Context) scanDir(root string, parentIgnore *Ignore) {
}
}

func (context *Context) UseStoreDb(pathList []string) (root string, relativePathList []string, err error) {
func (context *Context) UseAtomStore(root string, pathList []string) (relativePathList []string, err error) {

if len(pathList) == 0 {
return "", nil, errors.New("missing path(s)")
}
root, err = LocateIndexDb(pathList[0], context.IndexFilename)
if err == nil {

for _, path := range pathList {
path, err = filepath.Abs(path)
if err != nil {
return "", nil, err
}

// below root?
if !strings.HasPrefix(path, root) {
return "", nil, fmt.Errorf("path %s is not below the store-db in %s", path, root)
}
for _, path := range pathList {
path, err = filepath.Abs(path)
if err != nil {
return nil, err
}

relativePath, err := filepath.Rel(root, path)
if err != nil {
return "", nil, err
}
relativePathList = append(relativePathList, relativePath)
// below root?
if !strings.HasPrefix(path, root) {
return nil, fmt.Errorf("path %s is not below the atom store in %s", path, root)
}

context.store.UseDb(root, context.IndexFilename, len(relativePathList) == 1 && relativePathList[0] == ".")
relativePath, err := filepath.Rel(root, path)
if err != nil {
return nil, err
}
relativePathList = append(relativePathList, relativePath)
}

context.store.UseAtom(root, context.IndexFilename, len(relativePathList) == 1 && relativePathList[0] == ".")

return
}
2 changes: 1 addition & 1 deletion index.go
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ func (i *Index) calcFile(name string, a string) (*idxInfo, error) {
}

func (i *Index) save() (bool, error) {
if i.modified || !i.readonly && i.context.store.refreshDb {
if i.modified || !i.readonly && i.context.store.refresh {
if i.readonly {
return false, errors.New("error trying to save a readonly index")
}
Expand Down
10 changes: 10 additions & 0 deletions scripts/maketestsample
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/bin/bash
set -e

script_dir=$(dirname "$(realpath "$0")")

go run $script_dir/maketestsample.go -root /tmp/sample

echo
echo '$ ls -l /tmp/sample/root'
ls -l /tmp/sample/root
131 changes: 131 additions & 0 deletions scripts/maketestsample.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
package main

import (
"flag"
"fmt"
"os"
"path/filepath"
"time"
)

// perform integration test using the compiled binary

// misc files

var (
startList = []string{"time", "year", "people", "way", "day", "thing"}
wordList = []string{"life", "world", "school", "state", "family", "student", "group", "country", "problem", "hand", "part", "place", "case", "week", "company", "system", "program", "work", "government", "number", "night", "point", "home", "water", "room", "mother", "area", "money", "story", "fact", "month", "lot", "right", "study", "book", "eye", "job", "word", "business", "issue", "side", "kind", "head", "house", "service", "friend", "father", "power", "hour", "game", "line", "end", "member", "law", "car", "city", "community", "name", "president", "team", "minute", "idea", "kid", "body", "information", "back", "face", "others", "level", "office", "door", "health", "person", "art", "war", "history", "party", "result", "change", "morning", "reason", "research", "moment", "air", "teacher", "force", "education"}
extList = []string{"txt", "md", "pdf", "jpg", "jpeg", "png", "mp4", "mp3", "csv"}
startDate = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
endDate = time.Date(2024, 12, 1, 0, 0, 0, 0, time.UTC)
dateList = []time.Time{}
)

type genContext struct {
wordIdx int
extIdx int
dateIdx int
}

func init() {
var c int64 = 50
interval := (int64)(endDate.Sub(startDate).Seconds()) / c
for i := range make([]int64, c) {
dateList = append(dateList, startDate.Add(time.Duration(interval*(int64)(i))*time.Second))
}
}

func (g *genContext) nextWord() string {
word := wordList[g.wordIdx%len(wordList)]
g.wordIdx++
return word
}

func (g *genContext) nextExt() string {
ext := extList[g.extIdx%len(extList)]
g.extIdx++
return ext
}

func (g *genContext) setDate(filename string, r int) {
date := dateList[g.dateIdx%len(dateList)]
m := 17 * g.dateIdx / len(dateList)
date = date.Add(time.Duration(m) * time.Hour)
g.dateIdx++
os.Chtimes(filename, date, date)
}

func (g *genContext) genFile(path string, size int) {
os.WriteFile(path, make([]byte, size), 0644)
g.setDate(path, size*size)
}

func (g *genContext) genFiles(dir string, a int) {
os.MkdirAll(dir, 0755)
for i := 1; i <= 5; i++ {
size := a*i*g.wordIdx*100 + g.extIdx
file := g.nextWord() + "-" + g.nextWord()

if i%3 == 0 {
file += "-" + g.nextWord()
}

file += "." + g.nextExt()
g.genFile(filepath.Join(dir, file), size)
}
}

func (g *genContext) genDir(root string) {
for _, start := range startList {

for i := 1; i <= 5; i++ {
dir := filepath.Join(root, start, g.nextWord())
g.genFiles(dir, 1)

if g.wordIdx%3 == 0 {
dir = filepath.Join(dir, g.nextWord())
g.genFiles(dir, 1)
}
}
}
}

func (g *genContext) makeTestSampleFiles(testDir string) {

if err := os.RemoveAll(testDir); err != nil {
fmt.Println("Failed to clean", err)
panic(err)
}

root := filepath.Join(testDir, "root")
g.genDir(root)

os.MkdirAll(filepath.Join(root, "day/car/empty"), 0755)

rootPeople := filepath.Join(root, "people")
testPeople := filepath.Join(testDir, "people")

err := os.Rename(rootPeople, testPeople)
if err != nil {
fmt.Println("Rename failed", err)
panic(err)
}

err = os.Symlink(testPeople, rootPeople)
if err != nil {
fmt.Println("Symlink failed", err)
panic(err)
}
}

func main() {
root := flag.String("root", "", "root path to sample data (will be cleared)")
flag.Parse()
if *root == "" {
fmt.Println("error: root parameter is required")
os.Exit(1)
}
fmt.Printf("Clearing and generating test data in %s\n", *root)
g := genContext{}
g.makeTestSampleFiles(*root)
}
Loading

0 comments on commit 1577cc2

Please sign in to comment.