Continue transferring on list errors

2 files changed, 90 insertions, 38 deletions

diff --git a/cmd/lftpq/main.go b/cmd/lftpq/main.go
index afe55fa..52f6230 100644
--- a/cmd/lftpq/main.go
+++ b/cmd/lftpq/main.go
@@ -42,55 +42,62 @@ func (c *CLI) Run() error {
 		fmt.Fprintf(c.wr, "%s\n", json)
 		return nil
 	}
+	var queues []queue.Queue
 	if c.Import {
-		return c.processImportedQueue(cfg)
+		queues, err = queue.Read(cfg.Sites, c.rd)
 
+	} else {
+		queues, err = c.queueFor(cfg.Sites)
 	}
-	for _, s := range cfg.Sites {
-		if err := c.processQueue(s); err != nil {
-			fmt.Fprintf(c.wr, "error while processing queue for %s: %s\n", s.Name, err)
-		}
-	}
-	return nil
-}
-
-func (c *CLI) printf(format string, v ...interface{}) {
-	if !c.Quiet {
-		fmt.Fprintf(c.wr, format, v...)
-	}
-}
-
-func (c *CLI) processImportedQueue(cfg queue.Config) error {
-	queues, err := queue.Read(cfg.Sites, c.rd)
 	if err != nil {
 		return err
 	}
 	for _, q := range queues {
-		if err := c.process(q); err != nil {
-			return err
+		if err := c.transfer(q); err != nil {
+			c.printf("error while processing queue for %s: %s\n", q.Site.Name, err)
+			continue
 		}
 	}
 	return nil
 }
 
-func (c *CLI) processQueue(s queue.Site) error {
-	if s.Skip {
-		c.printf("[%s] Skipping site (Skip=%t)\n", s.Name, s.Skip)
-		return nil
+func (c *CLI) printf(format string, vs ...interface{}) {
+	alwaysPrint := false
+	for _, v := range vs {
+		if _, ok := v.(error); ok {
+			alwaysPrint = true
+			break
+		}
 	}
-	var files []os.FileInfo
-	for _, dir := range s.Dirs {
-		f, err := c.lister.List(s.Name, dir)
-		if err != nil {
-			return err
+	if !c.Quiet || alwaysPrint {
+		fmt.Fprint(c.wr, "lftpq: ")
+		fmt.Fprintf(c.wr, format, vs...)
+	}
+}
+
+func (c *CLI) queueFor(sites []queue.Site) ([]queue.Queue, error) {
+	var queues []queue.Queue
+	for _, s := range sites {
+		if s.Skip {
+			c.printf("skipping site %s\n", s.Name)
+			continue
+		}
+		var files []os.FileInfo
+		for _, dir := range s.Dirs {
+			f, err := c.lister.List(s.Name, dir)
+			if err != nil {
+				c.printf("error while listing %s on %s: %s\n", dir, s.Name, err)
+				continue
+			}
+			files = append(files, f...)
 		}
-		files = append(files, f...)
+		queue := queue.New(s, files)
+		queues = append(queues, queue)
 	}
-	queue := queue.New(s, files)
-	return c.process(queue)
+	return queues, nil
 }
 
-func (c *CLI) process(q queue.Queue) error {
+func (c *CLI) transfer(q queue.Queue) error {
 	if c.Dryrun {
 		var (
 			out []byte
@@ -108,7 +115,7 @@ func (c *CLI) process(q queue.Queue) error {
 		return err
 	}
 	if len(q.Transferable()) == 0 {
-		c.printf("[%s] Queue is empty\n", q.Site.Name)
+		c.printf("%s queue is empty\n", q.Site.Name)
 		return nil
 	}
 	if err := q.Start(c.consumer); err != nil {
diff --git a/cmd/lftpq/main_test.go b/cmd/lftpq/main_test.go
index 4501c23..0a69a89 100644
--- a/cmd/lftpq/main_test.go
+++ b/cmd/lftpq/main_test.go
@@ -25,6 +25,7 @@ func (f file) Sys() interface{}   { return nil }
 
 type testClient struct {
 	consumeQueue bool
+	failDirs     []string
 	dirList      []os.FileInfo
 }
 
@@ -35,7 +36,14 @@ func (c *testClient) Consume(path string) error {
 	return nil
 }
 
-func (c *testClient) List(name, path string) ([]os.FileInfo, error) { return c.dirList, nil }
+func (c *testClient) List(name, path string) ([]os.FileInfo, error) {
+	for _, d := range c.failDirs {
+		if d == path {
+			return nil, fmt.Errorf("read error")
+		}
+	}
+	return c.dirList, nil
+}
 
 func writeTestConfig(config string) (string, error) {
 	f, err := ioutil.TempFile("", "lftpq")
@@ -236,17 +244,16 @@ func TestRun(t *testing.T) {
 	defer os.Remove(cli.Config)
 
 	// Empty queue
-	client := testClient{consumeQueue: true}
 	if err := cli.Run(); err != nil {
 		t.Fatal(err)
 	}
-	want := "[t1] Queue is empty\n"
+	want := "lftpq: t1 queue is empty\n"
 	if got := buf.String(); got != want {
 		t.Errorf("want %q, got %q", want, got)
 	}
 
 	// Queue is consumed by client
-	client = testClient{consumeQueue: true, dirList: []os.FileInfo{file{name: "/baz/foo.2017"}}}
+	client := testClient{consumeQueue: true, dirList: []os.FileInfo{file{name: "/baz/foo.2017"}}}
 	cli.consumer = &client
 	cli.lister = &client
 	if err := cli.Run(); err != nil {
@@ -315,7 +322,45 @@ func TestRunSkipSite(t *testing.T) {
 	if err := cli.Run(); err != nil {
 		t.Fatal(err)
 	}
-	want := "[t1] Skipping site (Skip=true)\n"
+	want := "lftpq: skipping site t1\n"
+	if got := buf.String(); got != want {
+		t.Errorf("want %q, got %q", want, got)
+	}
+}
+
+func TestRunListError(t *testing.T) {
+	cli, buf := newTestCLI(`
+{
+   "Sites": [
+    {
+      "MaxAge": "0",
+      "Name": "t1",
+      "Dirs": [
+        "/foo"
+      ]
+    }
+   ]
+}`)
+	defer os.Remove(cli.Config)
+
+	client := testClient{failDirs: []string{"/foo"}}
+	cli.consumer = &client
+	cli.lister = &client
+	if err := cli.Run(); err != nil {
+		t.Fatal(err)
+	}
+	want := "lftpq: error while listing /foo on t1: read error\nlftpq: t1 queue is empty\n"
+	if got := buf.String(); got != want {
+		t.Errorf("want %q, got %q", want, got)
+	}
+
+	// Prints only error when quiet
+	buf.Reset()
+	cli.Quiet = true
+	if err := cli.Run(); err != nil {
+		t.Fatal(err)
+	}
+	want = "lftpq: error while listing /foo on t1: read error\n"
 	if got := buf.String(); got != want {
 		t.Errorf("want %q, got %q", want, got)
 	}