From 1577cc2251134f7a00aa43e6406e260def2030a1 Mon Sep 17 00:00:00 2001 From: Christian Zangl Date: Tue, 31 Dec 2024 12:15:46 +0100 Subject: [PATCH] split, atom mode for init --- README.md | 21 +++-- cmd/chkbit/help.go | 11 +-- cmd/chkbit/main.go | 41 ++++++---- context.go | 44 +++++------ index.go | 2 +- scripts/maketestsample | 10 +++ scripts/maketestsample.go | 131 +++++++++++++++++++++++++++++++ scripts/run_test.go | 157 ++++++++++++-------------------------- scripts/tests | 2 +- status.go | 6 +- store.go | 137 +++++++++++++++++++++------------ 11 files changed, 341 insertions(+), 221 deletions(-) create mode 100755 scripts/maketestsample create mode 100644 scripts/maketestsample.go diff --git a/README.md b/README.md index ecd29e7..3c9dd7a 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,6 @@ intact over time, especially during transfers and backups. Flags: -h, --help Show context-sensitive help. - --db use a index database instead of index files -m, --[no-]show-missing show missing files/directories -d, --[no-]include-dot include dot files -S, --[no-]skip-symlinks do not follow symlinks @@ -117,13 +116,10 @@ Commands: chkbit will verify files in readonly mode update ... [flags] - add and update indices + add and update indices (see flags with -h) - init-db [flags] - initialize a new index database at the given path for use with --db - - export-db [flags] - export a database to a json for archiving + init [flags] + initialize a new index at the given path show-ignored-only ... [flags] only show ignored files @@ -152,15 +148,16 @@ $ chkbit tips - lines starting with '/' are only applied to the current directory Status codes: + PNC: exception/panic, unable to continue DMG: error, data damage detected - EIX: error, index damaged + ERX: error, index damaged old: warning, file replaced by an older version - new: new file upd: file updated - ok : check ok - del: file/directory removed + new: new file + ok : checked and ok (verbose) + del: file/directory removed (-m) ign: ignored (see .chkbitignore) - EXC: exception/panic + msg: message Configuration file (json): - location /home/spark/.config/chkbit/config.json diff --git a/cmd/chkbit/help.go b/cmd/chkbit/help.go index a1c3147..8f59380 100644 --- a/cmd/chkbit/help.go +++ b/cmd/chkbit/help.go @@ -19,15 +19,16 @@ var helpTips = ` - lines starting with '/' are only applied to the current directory Status codes: + PNC: exception/panic, unable to continue DMG: error, data damage detected - EIX: error, index damaged + ERX: error, index damaged old: warning, file replaced by an older version - new: new file upd: file updated - ok : check ok - del: file/directory removed + new: new file + ok : checked and ok (verbose) + del: file/directory removed (-m) ign: ignored (see .chkbitignore) - EXC: exception/panic + msg: message Configuration file (json): - location diff --git a/cmd/chkbit/main.go b/cmd/chkbit/main.go index d1a88d3..904e2cb 100644 --- a/cmd/chkbit/main.go +++ b/cmd/chkbit/main.go @@ -60,12 +60,13 @@ type CLI struct { Paths []string `arg:"" name:"paths" help:"directories to update"` AddOnly bool `short:"a" help:"only add new and modified files, do not check existing (quicker)"` Force bool `help:"force update of damaged items (advanced usage only)"` - } `cmd:"" help:"add and update indices"` + } `cmd:"" help:"add and update indices (see flags with -h)"` - InitDb struct { - Path string `arg:"" help:"directory for the database"` - Force bool `help:"force init if a database already exists"` - } `cmd:"" help:"initialize a new index database at the given path for use with --db"` + Init struct { + Mode string `arg:"" enum:"split,atom" help:"the mode defines if you wish to store on index per directory (split) or one index in the given path (atom)"` + Path string `arg:"" help:"directory for the store root"` + Force bool `help:"force init if a store already exists"` + } `cmd:"" help:"initialize a new index at the given path"` ShowIgnoredOnly struct { Paths []string `arg:"" name:"paths" help:"directories to list"` @@ -77,7 +78,6 @@ type CLI struct { Version struct { } `cmd:"" help:"show version information"` - Db bool `help:"use a index database instead of index files"` ShowMissing bool `short:"m" help:"show missing files/directories" negatable:""` IncludeDot bool `short:"d" help:"include dot files" negatable:""` SkipSymlinks bool `short:"S" help:"do not follow symlinks" negatable:""` @@ -257,15 +257,20 @@ func (m *Main) process(cmd Command, cli CLI) (bool, error) { m.context.SkipSubdirectories = cli.NoRecurse m.context.TrackDirectories = !cli.NoDirInIndex - if cli.Db { - var root string - root, pathList, err = m.context.UseStoreDb(pathList) + st, root, err := chkbit.LocateStore(pathList[0], chkbit.StoreTypeAny, m.context.IndexFilename) + if err != nil { + return false, err + } + + if st == chkbit.StoreTypeAtom { + pathList, err = m.context.UseAtomStore(root, pathList) if err == nil { // pathList is relative to root - err = os.Chdir(root) - m.logInfo("", "Using store-db in "+root) - } - if err != nil { + if err = os.Chdir(root); err != nil { + return false, err + } + m.logInfo("", "Using atom-store in "+root) + } else { return false, err } } @@ -394,9 +399,13 @@ func (m *Main) run() int { cmd = Update case "show-ignored-only ": cmd = Show - case "init-db ": - m.logInfo("", "chkbit init-db "+cli.InitDb.Path) - if err := chkbit.InitializeIndexDb(cli.InitDb.Path, cli.IndexName, cli.InitDb.Force); err != nil { + case "init ": + m.logInfo("", fmt.Sprintf("chkbit init %s %s", cli.Init.Mode, cli.Init.Path)) + st := chkbit.StoreTypeSplit + if cli.Init.Mode == "atom" { + st = chkbit.StoreTypeAtom + } + if err := chkbit.InitializeStore(st, cli.Init.Path, cli.IndexName, cli.Init.Force); err != nil { m.logError(err.Error()) return 1 } diff --git a/context.go b/context.go index 56c0d35..3c07d51 100644 --- a/context.go +++ b/context.go @@ -113,7 +113,7 @@ func (context *Context) endWork() { } func (context *Context) isChkbitFile(name string) bool { - // any file with the index prefix is ignored (to allow for .bak and db files) + // any file with the index prefix is ignored (to allow for .bak and -db files) return strings.HasPrefix(name, context.IndexFilename) || name == context.IgnoreFilename } @@ -152,7 +152,8 @@ func (context *Context) Process(pathList []string) { if updated, err := context.store.Finish(); err != nil { context.logErr("index", err) } else if updated { - context.log(StatusInfo, "The index db was updated") + // todo + // context.log(StatusInfo, "The index store was updated") } context.LogQueue <- nil } @@ -209,34 +210,27 @@ func (context *Context) scanDir(root string, parentIgnore *Ignore) { } } -func (context *Context) UseStoreDb(pathList []string) (root string, relativePathList []string, err error) { +func (context *Context) UseAtomStore(root string, pathList []string) (relativePathList []string, err error) { - if len(pathList) == 0 { - return "", nil, errors.New("missing path(s)") - } - root, err = LocateIndexDb(pathList[0], context.IndexFilename) - if err == nil { - - for _, path := range pathList { - path, err = filepath.Abs(path) - if err != nil { - return "", nil, err - } - - // below root? - if !strings.HasPrefix(path, root) { - return "", nil, fmt.Errorf("path %s is not below the store-db in %s", path, root) - } + for _, path := range pathList { + path, err = filepath.Abs(path) + if err != nil { + return nil, err + } - relativePath, err := filepath.Rel(root, path) - if err != nil { - return "", nil, err - } - relativePathList = append(relativePathList, relativePath) + // below root? + if !strings.HasPrefix(path, root) { + return nil, fmt.Errorf("path %s is not below the atom store in %s", path, root) } - context.store.UseDb(root, context.IndexFilename, len(relativePathList) == 1 && relativePathList[0] == ".") + relativePath, err := filepath.Rel(root, path) + if err != nil { + return nil, err + } + relativePathList = append(relativePathList, relativePath) } + context.store.UseAtom(root, context.IndexFilename, len(relativePathList) == 1 && relativePathList[0] == ".") + return } diff --git a/index.go b/index.go index cc1e4c1..769df9b 100644 --- a/index.go +++ b/index.go @@ -223,7 +223,7 @@ func (i *Index) calcFile(name string, a string) (*idxInfo, error) { } func (i *Index) save() (bool, error) { - if i.modified || !i.readonly && i.context.store.refreshDb { + if i.modified || !i.readonly && i.context.store.refresh { if i.readonly { return false, errors.New("error trying to save a readonly index") } diff --git a/scripts/maketestsample b/scripts/maketestsample new file mode 100755 index 0000000..2a2506e --- /dev/null +++ b/scripts/maketestsample @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +script_dir=$(dirname "$(realpath "$0")") + +go run $script_dir/maketestsample.go -root /tmp/sample + +echo +echo '$ ls -l /tmp/sample/root' +ls -l /tmp/sample/root diff --git a/scripts/maketestsample.go b/scripts/maketestsample.go new file mode 100644 index 0000000..a1c2f74 --- /dev/null +++ b/scripts/maketestsample.go @@ -0,0 +1,131 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "time" +) + +// perform integration test using the compiled binary + +// misc files + +var ( + startList = []string{"time", "year", "people", "way", "day", "thing"} + wordList = []string{"life", "world", "school", "state", "family", "student", "group", "country", "problem", "hand", "part", "place", "case", "week", "company", "system", "program", "work", "government", "number", "night", "point", "home", "water", "room", "mother", "area", "money", "story", "fact", "month", "lot", "right", "study", "book", "eye", "job", "word", "business", "issue", "side", "kind", "head", "house", "service", "friend", "father", "power", "hour", "game", "line", "end", "member", "law", "car", "city", "community", "name", "president", "team", "minute", "idea", "kid", "body", "information", "back", "face", "others", "level", "office", "door", "health", "person", "art", "war", "history", "party", "result", "change", "morning", "reason", "research", "moment", "air", "teacher", "force", "education"} + extList = []string{"txt", "md", "pdf", "jpg", "jpeg", "png", "mp4", "mp3", "csv"} + startDate = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + endDate = time.Date(2024, 12, 1, 0, 0, 0, 0, time.UTC) + dateList = []time.Time{} +) + +type genContext struct { + wordIdx int + extIdx int + dateIdx int +} + +func init() { + var c int64 = 50 + interval := (int64)(endDate.Sub(startDate).Seconds()) / c + for i := range make([]int64, c) { + dateList = append(dateList, startDate.Add(time.Duration(interval*(int64)(i))*time.Second)) + } +} + +func (g *genContext) nextWord() string { + word := wordList[g.wordIdx%len(wordList)] + g.wordIdx++ + return word +} + +func (g *genContext) nextExt() string { + ext := extList[g.extIdx%len(extList)] + g.extIdx++ + return ext +} + +func (g *genContext) setDate(filename string, r int) { + date := dateList[g.dateIdx%len(dateList)] + m := 17 * g.dateIdx / len(dateList) + date = date.Add(time.Duration(m) * time.Hour) + g.dateIdx++ + os.Chtimes(filename, date, date) +} + +func (g *genContext) genFile(path string, size int) { + os.WriteFile(path, make([]byte, size), 0644) + g.setDate(path, size*size) +} + +func (g *genContext) genFiles(dir string, a int) { + os.MkdirAll(dir, 0755) + for i := 1; i <= 5; i++ { + size := a*i*g.wordIdx*100 + g.extIdx + file := g.nextWord() + "-" + g.nextWord() + + if i%3 == 0 { + file += "-" + g.nextWord() + } + + file += "." + g.nextExt() + g.genFile(filepath.Join(dir, file), size) + } +} + +func (g *genContext) genDir(root string) { + for _, start := range startList { + + for i := 1; i <= 5; i++ { + dir := filepath.Join(root, start, g.nextWord()) + g.genFiles(dir, 1) + + if g.wordIdx%3 == 0 { + dir = filepath.Join(dir, g.nextWord()) + g.genFiles(dir, 1) + } + } + } +} + +func (g *genContext) makeTestSampleFiles(testDir string) { + + if err := os.RemoveAll(testDir); err != nil { + fmt.Println("Failed to clean", err) + panic(err) + } + + root := filepath.Join(testDir, "root") + g.genDir(root) + + os.MkdirAll(filepath.Join(root, "day/car/empty"), 0755) + + rootPeople := filepath.Join(root, "people") + testPeople := filepath.Join(testDir, "people") + + err := os.Rename(rootPeople, testPeople) + if err != nil { + fmt.Println("Rename failed", err) + panic(err) + } + + err = os.Symlink(testPeople, rootPeople) + if err != nil { + fmt.Println("Symlink failed", err) + panic(err) + } +} + +func main() { + root := flag.String("root", "", "root path to sample data (will be cleared)") + flag.Parse() + if *root == "" { + fmt.Println("error: root parameter is required") + os.Exit(1) + } + fmt.Printf("Clearing and generating test data in %s\n", *root) + g := genContext{} + g.makeTestSampleFiles(*root) +} diff --git a/scripts/run_test.go b/scripts/run_test.go index fd7ed1a..bfd7530 100644 --- a/scripts/run_test.go +++ b/scripts/run_test.go @@ -13,7 +13,7 @@ import ( // perform integration test using the compiled binary -var testDir = "/tmp/chkbit" +const testDirBase = "/tmp/chkbit" func runCmd(args ...string) *exec.Cmd { _, filename, _, _ := runtime.Caller(0) @@ -35,113 +35,32 @@ func checkNotOut(t *testing.T, sout string, notExpected string) { } } -// misc files - -var ( - startList = []string{"time", "year", "people", "way", "day", "thing"} - wordList = []string{"life", "world", "school", "state", "family", "student", "group", "country", "problem", "hand", "part", "place", "case", "week", "company", "system", "program", "work", "government", "number", "night", "point", "home", "water", "room", "mother", "area", "money", "story", "fact", "month", "lot", "right", "study", "book", "eye", "job", "word", "business", "issue", "side", "kind", "head", "house", "service", "friend", "father", "power", "hour", "game", "line", "end", "member", "law", "car", "city", "community", "name", "president", "team", "minute", "idea", "kid", "body", "information", "back", "face", "others", "level", "office", "door", "health", "person", "art", "war", "history", "party", "result", "change", "morning", "reason", "research", "moment", "air", "teacher", "force", "education"} - extList = []string{"txt", "md", "pdf", "jpg", "jpeg", "png", "mp4", "mp3", "csv"} - startDate = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) - endDate = time.Date(2024, 12, 1, 0, 0, 0, 0, time.UTC) - dateList = []time.Time{} - wordIdx = 0 - extIdx = 0 - dateIdx = 0 -) - -func nextWord() string { - word := wordList[wordIdx%len(wordList)] - wordIdx++ - return word -} - -func nextExt() string { - ext := extList[extIdx%len(extList)] - extIdx++ - return ext -} - -func setDate(filename string, r int) { - date := dateList[dateIdx%len(dateList)] - m := 17 * dateIdx / len(dateList) - date = date.Add(time.Duration(m) * time.Hour) - dateIdx++ - os.Chtimes(filename, date, date) -} - -func genFile(path string, size int) { - os.WriteFile(path, make([]byte, size), 0644) - setDate(path, size*size) -} - -func genFiles(dir string, a int) { - os.MkdirAll(dir, 0755) - for i := 1; i <= 5; i++ { - size := a*i*wordIdx*100 + extIdx - file := nextWord() + "-" + nextWord() - - if i%3 == 0 { - file += "-" + nextWord() - } - - file += "." + nextExt() - genFile(filepath.Join(dir, file), size) - } -} - -func genDir(root string) { - for _, start := range startList { - - for i := 1; i <= 5; i++ { - dir := filepath.Join(root, start, nextWord()) - genFiles(dir, 1) - - if wordIdx%3 == 0 { - dir = filepath.Join(dir, nextWord()) - genFiles(dir, 1) - } +func initStore(t *testing.T, storeType, root string) { + t.Run("init", func(t *testing.T) { + cmd := runCmd("init", storeType, root) + out, err := cmd.Output() + if err != nil { + t.Fatalf("failed with '%s'\n", err) } - } + sout := string(out) + checkOut(t, sout, "chkbit init "+storeType) + checkNotOut(t, sout, "EXC") + }) } -func setupMiscFiles() { - - var c int64 = 50 - interval := (int64)(endDate.Sub(startDate).Seconds()) / c - for i := range make([]int64, c) { - dateList = append(dateList, startDate.Add(time.Duration(interval*(int64)(i))*time.Second)) - } +func testRoot(t *testing.T, storeType string) { + testDir := filepath.Join(testDirBase, storeType) root := filepath.Join(testDir, "root") - if err := os.RemoveAll(testDir); err != nil { - fmt.Println("Failed to clean", err) - panic(err) - } - - genDir(root) + g := genContext{} + g.makeTestSampleFiles(testDir) - os.MkdirAll(filepath.Join(root, "day/car/empty"), 0755) - - rootPeople := filepath.Join(root, "people") - testPeople := filepath.Join(testDir, "people") - - err := os.Rename(rootPeople, testPeople) - if err != nil { - fmt.Println("Rename failed", err) - panic(err) - } - - err = os.Symlink(testPeople, rootPeople) - if err != nil { - fmt.Println("Symlink failed", err) - panic(err) + checkPrefix := "/tmp/chkbit/split/root/" + if storeType == "atom" { + checkPrefix = "" } -} - -func TestRoot(t *testing.T) { - setupMiscFiles() - root := filepath.Join(testDir, "root") + initStore(t, storeType, root) // update index, no recourse t.Run("no-recourse", func(t *testing.T) { @@ -184,7 +103,7 @@ func TestRoot(t *testing.T) { t.Fatalf("failed with '%s'\n", err) } sout := string(out) - checkOut(t, sout, "del /tmp/chkbit/root/thing/change/") + checkOut(t, sout, "del "+checkPrefix+"thing/change/") checkOut(t, sout, "2 files/directories would have been removed") }) @@ -208,7 +127,7 @@ func TestRoot(t *testing.T) { t.Fatalf("failed with '%s'\n", err) } sout := string(out) - checkOut(t, sout, "del /tmp/chkbit/root/thing/change/") + checkOut(t, sout, "del "+checkPrefix+"thing/change/") checkOut(t, sout, "2 files/directories have been removed") }) @@ -231,8 +150,8 @@ func TestRoot(t *testing.T) { // add files only t.Run("add-only", func(t *testing.T) { - genFiles(filepath.Join(root, "way/add"), 99) - genFile(filepath.Join(root, "time/add-file.txt"), 500) + g.genFiles(filepath.Join(root, "way/add"), 99) + g.genFile(filepath.Join(root, "time/add-file.txt"), 500) cmd := runCmd("update", "-a", root) out, err := cmd.Output() @@ -250,7 +169,7 @@ func TestRoot(t *testing.T) { t.Run("add-only-mod", func(t *testing.T) { // modify existing - genFile(filepath.Join(root, "way/job/word-business.mp3"), 500) + g.genFile(filepath.Join(root, "way/job/word-business.mp3"), 500) cmd := runCmd("update", "-a", root) out, err := cmd.Output() @@ -258,7 +177,7 @@ func TestRoot(t *testing.T) { t.Fatalf("failed with '%s'\n", err) } sout := string(out) - checkOut(t, sout, "old /tmp/chkbit/root/way/job/word-business.mp3") + checkOut(t, sout, "old "+checkPrefix+"way/job/word-business.mp3") checkOut(t, sout, "Processed 1 file") checkOut(t, sout, "- 1 directory was updated") checkOut(t, sout, "- 0 file hashes were added") @@ -279,8 +198,8 @@ func TestRoot(t *testing.T) { // ignore dot t.Run("ignore-dot", func(t *testing.T) { - genFiles(filepath.Join(root, "way/.hidden"), 99) - genFile(filepath.Join(root, "time/.ignored"), 999) + g.genFiles(filepath.Join(root, "way/.hidden"), 99) + g.genFile(filepath.Join(root, "time/.ignored"), 999) cmd := runCmd("update", root) out, err := cmd.Output() @@ -307,9 +226,9 @@ func TestRoot(t *testing.T) { }) } -func TestDMG(t *testing.T) { +func testDMG(t *testing.T, storeType string) { - testDmg := filepath.Join(testDir, "test_dmg") + testDmg := filepath.Join(testDirBase, "test_dmg", storeType) if err := os.RemoveAll(testDmg); err != nil { fmt.Println("Failed to clean", err) panic(err) @@ -324,6 +243,8 @@ func TestDMG(t *testing.T) { panic(err) } + initStore(t, storeType, ".") + testFile := filepath.Join(testDmg, "test.txt") t1, _ := time.Parse(time.RFC3339, "2022-02-01T11:00:00Z") t2, _ := time.Parse(time.RFC3339, "2022-02-01T12:00:00Z") @@ -384,3 +305,19 @@ func TestDMG(t *testing.T) { } }) } + +func TestRootAtom(t *testing.T) { + testRoot(t, "atom") +} + +func TestRootSplit(t *testing.T) { + testRoot(t, "split") +} + +func TestDmgAtom(t *testing.T) { + testDMG(t, "atom") +} + +func TestDmgSplit(t *testing.T) { + testDMG(t, "split") +} diff --git a/scripts/tests b/scripts/tests index 736df2b..ad945ac 100755 --- a/scripts/tests +++ b/scripts/tests @@ -9,7 +9,7 @@ go test -v . echo "# test util" go test -v ./cmd/chkbit/util -count=1 -echo "# prep files" +echo "# build" $script_dir/build echo "# test files" diff --git a/status.go b/status.go index c4a63a2..7eac723 100644 --- a/status.go +++ b/status.go @@ -3,15 +3,15 @@ package chkbit type Status string const ( - StatusPanic Status = "EXC" - StatusErrorIdx Status = "ERX" + StatusPanic Status = "PNC" StatusErrorDamage Status = "DMG" + StatusErrorIdx Status = "ERX" StatusUpdateWarnOld Status = "old" StatusUpdate Status = "upd" StatusNew Status = "new" StatusOK Status = "ok " - StatusIgnore Status = "ign" StatusMissing Status = "del" + StatusIgnore Status = "ign" StatusInfo Status = "msg" // internal diff --git a/store.go b/store.go index c97285b..3f0c355 100644 --- a/store.go +++ b/store.go @@ -5,59 +5,69 @@ import ( "errors" "os" "path/filepath" + "slices" "sync" "time" bolt "go.etcd.io/bbolt" ) +type StoreType int + +const ( + StoreTypeAny StoreType = iota + StoreTypeSplit + StoreTypeAtom +) + type storeDbItem struct { key []byte value []byte } type store struct { + indexName string + logQueue chan *LogEvent + readOnly bool - useDb bool - refreshDb bool + atom bool + refresh bool dirty bool - dbPath string - indexName string - dbFile string + atomPath string cacheFileR string cacheFileW string connR *bolt.DB connW *bolt.DB storeDbQueue chan *storeDbItem storeDbWg sync.WaitGroup - logQueue chan *LogEvent } const ( dbSuffix = "-db" - bakDbSuffix = ".bak" - newDbSuffix = ".new" + bakSuffix = ".bak" + newSuffix = ".new" dbTxTimeoutSec = 30 - chkbitDbPrefix = `{"type":"chkbit","version":6,"data":{` - chkbitDbSuffix = `}}` + atomDataPrefix = `{"type":"chkbit","version":6,"data":{` + atomDataSuffix = `}}` ) -func (s *store) UseDb(path string, indexName string, refresh bool) { - s.dbPath = path +var storeTypeList = []StoreType{StoreTypeAtom, StoreTypeSplit} + +func (s *store) UseAtom(path string, indexName string, refresh bool) { + s.atomPath = path s.indexName = indexName - s.useDb = true - s.refreshDb = refresh + s.atom = true + s.refresh = refresh } func (s *store) logErr(message string) { - s.logQueue <- &LogEvent{StatusPanic, "index-db: " + message} + s.logQueue <- &LogEvent{StatusPanic, "store: " + message} } func (s *store) Open(readOnly bool, numWorkers int) error { var err error s.readOnly = readOnly - if s.useDb { - s.dbFile = getDbFile(s.dbPath, s.indexName, "") + if s.atom { if s.cacheFileR, err = getTempDbFile(s.indexName); err != nil { return err @@ -71,15 +81,15 @@ func (s *store) Open(readOnly bool, numWorkers int) error { if !readOnly { - // test if the new db file is writeable before failing at the end - testWrite := getDbFile(s.dbPath, s.indexName, newDbSuffix) + // test if the new store file is writeable before failing at the end + testWrite := getAtomFile(s.atomPath, s.indexName, newSuffix) if file, err := os.Create(testWrite); err != nil { return err } else { defer file.Close() } - if s.refreshDb { + if s.refresh { // write to a new db if s.cacheFileW, err = getTempDbFile(s.indexName); err != nil { return err @@ -106,7 +116,7 @@ func (s *store) Open(readOnly bool, numWorkers int) error { func (s *store) Finish() (updated bool, err error) { - if !s.useDb { + if !s.atom { return } @@ -119,7 +129,7 @@ func (s *store) Finish() (updated bool, err error) { if err = s.connR.Close(); err != nil { return } - if !s.readOnly && s.refreshDb { + if !s.readOnly && s.refresh { if err = s.connW.Close(); err != nil { return } @@ -136,14 +146,15 @@ func (s *store) Finish() (updated bool, err error) { } var newFile string - if newFile, err = s.exportCache(cacheFile, newDbSuffix); err != nil { + if newFile, err = s.exportCache(cacheFile, newSuffix); err != nil { return } - if err = os.Rename(s.dbFile, getDbFile(s.dbPath, s.indexName, bakDbSuffix)); err != nil { + atomFile := getAtomFile(s.atomPath, s.indexName, "") + if err = os.Rename(atomFile, getAtomFile(s.atomPath, s.indexName, bakSuffix)); err != nil { return } - if err = os.Rename(newFile, s.dbFile); err != nil { + if err = os.Rename(newFile, atomFile); err != nil { return } @@ -161,7 +172,7 @@ func (s *store) Finish() (updated bool, err error) { func (s *store) Load(indexPath string) ([]byte, error) { var err error var value []byte - if s.useDb { + if s.atom { if s.connR == nil { return nil, errors.New("db not loaded") } @@ -185,7 +196,7 @@ func (s *store) Load(indexPath string) ([]byte, error) { func (s *store) Save(indexPath string, value []byte) error { var err error s.dirty = true - if s.useDb { + if s.atom { s.storeDbQueue <- &storeDbItem{[]byte(indexPath), value} } else { // try to preserve the directory mod time but ignore if unsupported @@ -249,15 +260,15 @@ func (s *store) exportCache(dbFile, suffix string) (exportFile string, err error } defer connR.Close() - exportFile = getDbFile(s.dbPath, s.indexName, suffix) + exportFile = getAtomFile(s.atomPath, s.indexName, suffix) file, err := os.Create(exportFile) if err != nil { return } defer file.Close() - // export version 6 database - if _, err = file.WriteString(chkbitDbPrefix); err != nil { + // export version 6 store + if _, err = file.WriteString(atomDataPrefix); err != nil { return } @@ -302,7 +313,7 @@ func (s *store) exportCache(dbFile, suffix string) (exportFile string, err error return } - if _, err = file.WriteString(chkbitDbSuffix); err != nil { + if _, err = file.WriteString(atomDataSuffix); err != nil { return } @@ -323,7 +334,7 @@ func (s *store) importCache(dbFile string) error { return err } - file, err := os.Open(getDbFile(s.dbPath, s.indexName, "")) + file, err := os.Open(getAtomFile(s.atomPath, s.indexName, "")) if err != nil { return err } @@ -405,12 +416,31 @@ func (s *store) importCache(dbFile string) error { return err } -func getDbFile(path, indexFilename, suffix string) string { - return filepath.Join(path, indexFilename+dbSuffix+suffix) +func getAtomFile(path, indexName, suffix string) string { + return filepath.Join(path, indexName+dbSuffix+suffix) } -func getTempDbFile(indexFilename string) (string, error) { - tempFile, err := os.CreateTemp("", "*"+indexFilename) +func getMarkerFile(st StoreType, path, indexName string) string { + if st == StoreTypeSplit { + return filepath.Join(path, indexName) + } else { + return getAtomFile(path, indexName, "") + } +} + +func existsMarkerFile(st StoreType, path, indexName string) (ok bool, err error) { + fileName := getMarkerFile(st, path, indexName) + _, err = os.Stat(fileName) + if err == nil { + ok = true + } else if os.IsNotExist(err) { + err = nil + } + return +} + +func getTempDbFile(indexName string) (string, error) { + tempFile, err := os.CreateTemp("", "*"+indexName) if err == nil { tempFile.Close() } @@ -426,8 +456,11 @@ func getBoltOptions(readOnly bool) *bolt.Options { } } -func InitializeIndexDb(path, indexName string, force bool) error { - fileName := getDbFile(path, indexName, "") +func InitializeStore(st StoreType, path, indexName string, force bool) error { + if !slices.Contains(storeTypeList, st) { + return errors.New("invalid type") + } + fileName := getMarkerFile(st, path, indexName) _, err := os.Stat(fileName) if !os.IsNotExist(err) { if force { @@ -443,25 +476,33 @@ func InitializeIndexDb(path, indexName string, force bool) error { return err } defer file.Close() - _, err = file.WriteString(chkbitDbPrefix + chkbitDbSuffix) + init := atomDataPrefix + atomDataSuffix + if st == StoreTypeSplit { + init = "{}" + } + _, err = file.WriteString(init) return err } -func LocateIndexDb(path, indexName string) (string, error) { - var err error - if path, err = filepath.Abs(path); err != nil { - return "", err +func LocateStore(startPath string, filter StoreType, indexName string) (st StoreType, path string, err error) { + if path, err = filepath.Abs(startPath); err != nil { + return } for { - file := getDbFile(path, indexName, "") - _, err = os.Stat(file) - if !os.IsNotExist(err) { - return path, nil + var ok bool + for _, st = range storeTypeList { + if filter == StoreTypeAny || filter == st { + if ok, err = existsMarkerFile(st, path, indexName); ok || err != nil { + return + } + } } + path = filepath.Dir(path) if len(path) < 1 || path[len(path)-1] == filepath.Separator { // reached root - return "", errors.New("index db could not be located (forgot to initialize?)") + err = errors.New("index could not be located (see chkbit init)") + return } } }