Verify checksums incrementally

4 files changed, 137 insertions, 41 deletions

diff --git a/rar/rar.go b/rar/rar.go
index 115b43b..63c0c25 100644
--- a/rar/rar.go
+++ b/rar/rar.go
@@ -10,6 +10,7 @@ import (
 	"strings"
 	"sync"
 	"text/template"
+	"time"
 
 	"github.com/mpolden/sfv"
 	"github.com/nwaples/rardecode"
@@ -26,11 +27,11 @@ type event struct {
 }
 
 type Handler struct {
-	mu sync.Mutex
+	mu    sync.Mutex
+	cache map[string]bool
+	done  chan bool
 }
 
-func NewHandler() *Handler { return &Handler{} }
-
 func eventFrom(filename string) (event, error) {
 	dir := filepath.Dir(filename)
 	sfv, err := sfv.Find(dir)
@@ -142,29 +143,6 @@ func remove(sfv *sfv.SFV) error {
 	return os.Remove(sfv.Path)
 }
 
-func fileCount(sfv *sfv.SFV) (int, int) {
-	exists := 0
-	for _, c := range sfv.Checksums {
-		if c.IsExist() {
-			exists++
-		}
-	}
-	return exists, len(sfv.Checksums)
-}
-
-func verify(sfv *sfv.SFV) error {
-	for _, c := range sfv.Checksums {
-		ok, err := c.Verify()
-		if err != nil {
-			return err
-		}
-		if !ok {
-			return errors.Errorf("%s: failed checksum: %s", sfv.Path, c.Filename)
-		}
-	}
-	return nil
-}
-
 func cmdFrom(tmpl string, ev event) (*exec.Cmd, error) {
 	t, err := template.New("cmd").Parse(tmpl)
 	if err != nil {
@@ -199,6 +177,60 @@ func runCmd(command string, e event) error {
 	return nil
 }
 
+func NewHandler() *Handler { return NewHandlerWithInterval(time.Minute) }
+
+func NewHandlerWithInterval(d time.Duration) *Handler {
+	h := &Handler{
+		cache: make(map[string]bool),
+		done:  make(chan bool),
+	}
+	ticker := time.NewTicker(d)
+	go func() {
+		for {
+			select {
+			case <-h.done:
+				ticker.Stop()
+				return
+			case <-ticker.C:
+				h.pruneCache()
+			}
+		}
+
+	}()
+	return h
+}
+
+func (h *Handler) verify(sfv *sfv.SFV) (int, int, error) {
+	passed := 0
+	for _, c := range sfv.Checksums {
+		ok := h.cache[c.Path]
+		if !ok && c.IsExist() {
+			var err error
+			ok, err = c.Verify()
+			if err != nil {
+				return 0, 0, err
+			}
+			h.cache[c.Path] = ok
+		}
+		if ok {
+			passed++
+		}
+	}
+	return passed, len(sfv.Checksums), nil
+}
+
+func (h *Handler) pruneCache() {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	for path := range h.cache {
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			delete(h.cache, path)
+		}
+	}
+}
+
+func (h *Handler) Stop() { h.done <- true }
+
 func (h *Handler) Handle(name, postCommand string, removeRARs bool) error {
 	h.mu.Lock()
 	defer h.mu.Unlock()
@@ -206,12 +238,13 @@ func (h *Handler) Handle(name, postCommand string, removeRARs bool) error {
 	if err != nil {
 		return err
 	}
-	if exists, total := fileCount(ev.sfv); exists != total {
-		return errors.Errorf("incomplete: %s: %d/%d files", ev.Dir, exists, total)
-	}
-	if err := verify(ev.sfv); err != nil {
+	passed, total, err := h.verify(ev.sfv)
+	if err != nil {
 		return errors.Wrapf(err, "verification failed: %s", ev.Dir)
 	}
+	if passed != total {
+		return errors.Errorf("incomplete: %s: %d/%d files", ev.Dir, passed, total)
+	}
 	if err := unpack(ev.Name); err != nil {
 		return errors.Wrapf(err, "unpacking failed: %s", ev.Dir)
 	}
diff --git a/rar/rar_test.go b/rar/rar_test.go
index 96d88df..bfc863f 100644
--- a/rar/rar_test.go
+++ b/rar/rar_test.go
@@ -1,6 +1,7 @@
 package rar
 
 import (
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -10,6 +11,20 @@ import (
 	"github.com/mpolden/sfv"
 )
 
+func symlink(t *testing.T, oldname, newname string) {
+	if err := os.Symlink(oldname, newname); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testdataDir(t *testing.T) string {
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return filepath.Join(wd, "testdata")
+}
+
 func TestCmdFrom(t *testing.T) {
 	tmpl := "tar -xf {{.Name}} {{.Base}} {{.Dir}}"
 	values := event{
@@ -88,22 +103,18 @@ func TestFindFirstRAR(t *testing.T) {
 }
 
 func TestHandle(t *testing.T) {
-	wd, err := os.Getwd()
-	if err != nil {
-		t.Fatal(err)
-	}
-	testdata := filepath.Join(wd, "testdata")
+	td := testdataDir(t)
 
 	var tests = []struct {
 		file  string
 		mtime int64
 	}{
-		{filepath.Join(testdata, "test1"), 1498246676},
-		{filepath.Join(testdata, "test2"), 1498246678},
-		{filepath.Join(testdata, "test3"), 1498246680},
-		{filepath.Join(testdata, "test", "test4"), 1498316526},
-		{filepath.Join(testdata, "test"), 1498316526},
-		{filepath.Join(testdata, "nested.rar"), 1498246697},
+		{filepath.Join(td, "test1"), 1498246676},
+		{filepath.Join(td, "test2"), 1498246678},
+		{filepath.Join(td, "test3"), 1498246680},
+		{filepath.Join(td, "test", "test4"), 1498316526},
+		{filepath.Join(td, "test"), 1498316526},
+		{filepath.Join(td, "nested.rar"), 1498246697},
 	}
 
 	defer func() {
@@ -132,3 +143,51 @@ func TestHandle(t *testing.T) {
 		}
 	}
 }
+
+func TestHandleIncomplete(t *testing.T) {
+	var (
+		td       = testdataDir(t)
+		realRAR1 = filepath.Join(td, "test.rar")
+		realRAR2 = filepath.Join(td, "test.r00")
+		realSFV  = filepath.Join(td, "test.sfv")
+	)
+	tempdir, err := ioutil.TempDir("", "unp")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempdir)
+
+	var (
+		sfv  = filepath.Join(tempdir, "test.sfv")
+		rar1 = filepath.Join(tempdir, "test.rar")
+		rar2 = filepath.Join(tempdir, "test.r00")
+	)
+
+	symlink(t, realSFV, sfv)
+	symlink(t, realRAR1, rar1)
+	symlink(t, realRAR2, rar2)
+
+	h := NewHandlerWithInterval(time.Hour)
+	defer h.Stop()
+	want := "incomplete: " + tempdir + ": 2/3 files"
+	if err := h.Handle(rar1, "", false); err.Error() != want {
+		t.Errorf("want err = %q, got %q", want, err.Error())
+	}
+
+	// Removing a file keeps the cached checksum
+	if err := os.Remove(rar1); err != nil {
+		t.Fatal(err)
+	}
+	if err := h.Handle(rar1, "", false); err.Error() != want {
+		t.Errorf("want err = %q, got %q", want, err.Error())
+	}
+	if want, got := 2, len(h.cache); want != got {
+		t.Errorf("want len = %d, got %d", want, got)
+	}
+
+	// ... until cache is pruned
+	h.pruneCache()
+	if want, got := 1, len(h.cache); want != got {
+		t.Errorf("want len = %d, got %d", want, got)
+	}
+}
diff --git a/watcher/watcher.go b/watcher/watcher.go
index 7a87a21..6525118 100644
--- a/watcher/watcher.go
+++ b/watcher/watcher.go
@@ -17,6 +17,7 @@ import (
 
 type Handler interface {
 	Handle(filename, postCommand string, remove bool) error
+	Stop()
 }
 
 type Watcher struct {
@@ -151,6 +152,7 @@ func (w *Watcher) Start() {
 }
 
 func (w *Watcher) Stop() {
+	w.handler.Stop()
 	notify.Stop(w.events)
 	w.done <- true
 	w.done <- true
diff --git a/watcher/watcher_test.go b/watcher/watcher_test.go
index 2e811ce..94e3329 100644
--- a/watcher/watcher_test.go
+++ b/watcher/watcher_test.go
@@ -24,6 +24,8 @@ func (h *testHandler) Handle(filename, postCommand string, remove bool) error {
 	return nil
 }
 
+func (h *testHandler) Stop() {}
+
 func (h *testHandler) awaitFile(file string) (bool, error) {
 	ts := time.Now()
 	for len(h.files) == 0 {