Merge pull request #209 from sabjorn/feature/208-make-downloads-parallel

Feature/208 make downloads parallel

Author: sabjorn

css/style.css

@@ -274,3 +274,29 @@ input.currency-input::-webkit-outer-spin-button {
     padding-bottom: .5em;
     text-align: center;
 }
+
+button.bes-downloadall:hover {
+  text-decoration: none;
+}
+
+button.bes-downloadall[disabled] {
+  cursor: default;
+  opacity: 0.30;
+  color: #000000;
+}
+
+@keyframes loadingDots {
+  0% { content: ""; }
+  25% { content: "."; }
+  50% { content: ".."; }
+  75% { content: "..."; }
+  100% { content: ""; }
+}
+
+button.bes-downloadall[disabled]::after {
+  content: "";
+  display: inline-block;
+  animation: loadingDots 1.5s infinite;
+  width: 1.1em;
+  text-align: left;
+}

src/download_helper.js

@@ -2,16 +2,6 @@ import Logger from "./logger";
 
 import { downloadFile, dateString } from "./utilities";
 
-const preamble = `#!/usr/bin/env bash
-# Generated by Bandcamp Enhancement Suite (https://github.com/sabjorn/BandcampEnhancementSuite)
-#
-# The following can be used to batch download your recent purchase.
-#
-# Usage (Mac/Linux):
-# 1) open Terminal
-# 2) move to desired download directory (e.g. \`cd ~/Downloads/bandcamp\`)
-# 3) paste the text of this file into Terminal\n\n`;
-
 export default class DownloadHelper {
   constructor() {
     this.log = new Logger();
@@ -52,7 +42,7 @@ export default class DownloadHelper {
     this.button = document.createElement("button");
     this.button.title =
       "Generates a file for automating downloads using 'cURL'";
-    this.button.className = "downloadall";
+    this.button.className = "bes-downloadall";
     this.button.disabled = true;
     this.button.textContent = "preparing download";
 
@@ -69,7 +59,8 @@ export default class DownloadHelper {
       const date = DownloadHelper.dateString();
       const downloadList = DownloadHelper.generateDownloadList();
       const preamble = DownloadHelper.getDownloadPreamble();
-      const downloadDocument = preamble + downloadList;
+      const postamble = DownloadHelper.getDownloadPostamble();
+      const downloadDocument = preamble + downloadList + postamble;
 
       DownloadHelper.downloadFile(`bandcamp_${date}.txt`, downloadDocument);
     });
@@ -84,33 +75,28 @@ export default class DownloadHelper {
   }
 
   static generateDownloadList() {
-    let filelist = "";
-    document.querySelectorAll("a.item-button").forEach((item, index, list) => {
-      const url = item.getAttribute("href");
-      // Prevent duplicate URLs
-      if (filelist.indexOf(url) === -1) {
-        filelist += 'curl -OJ "' + url + '" && \\\n';
-      }
-    });
-    filelist = filelist.substring(0, filelist.length - 6);
-    filelist += "\n";
-    return filelist;
+    const urlSet = new Set(
+      [...document.querySelectorAll("a.item-button")].map(item => {
+        return item.getAttribute("href");
+      })
+    );
+
+    if (urlSet.size === 0) return "URLS=()\n";
+
+    const fileList = [...urlSet].map(url => `\t"${url}"`).join("\n");
+    return "URLS=(\n" + fileList + "\n)\n";
   }
 
   static callback() {
-    let allDownloadLinks = document.querySelectorAll(
+    const allDownloadLinks = document.querySelectorAll(
       ".download-title .item-button"
     );
 
-    let linksReady = true;
-    allDownloadLinks.forEach(function(element, index) {
-      if (element.style.display === "none") {
-        linksReady = false;
-        return;
-      }
-    });
+    const linksReady = [...allDownloadLinks].every(
+      element => element.style.display !== "none"
+    );
 
-    this.log.info("linksReady", linksReady);
+    this.log.info(`linksReady: ${linksReady}`);
     if (linksReady) {
       this.enableButton();
       return;
@@ -122,4 +108,77 @@ export default class DownloadHelper {
   static getDownloadPreamble() {
     return preamble;
   }
+
+  static getDownloadPostamble() {
+    return postamble;
+  }
 }
+
+const preamble = `#!/usr/bin/env bash
+
+# Generated by Bandcamp Enhancement Suite (https://github.com/sabjorn/BandcampEnhancementSuite)
+#
+# The following can be used to batch download your recent purchases.
+# NOTE: pasting into terminal may not work properly--please follow new instructions below
+#
+# Usage (Mac/Linux):
+# 1) open Terminal
+# 2) move to desired download directory (e.g. \`cd ~/Downloads/bandcamp\`)
+# 3) run script (e.g. \`bash <this filename>.txt\`
+
+`;
+
+const postamble = `
+DEFAULT_BATCH_SIZE=5
+
+download_file() {
+    local url="$1"
+    
+    if curl -L --fail -OJ "$url" 2>/dev/null; then
+        echo -n "."
+        return 0
+    else
+        echo -n "x"
+        return 1
+    fi
+}
+
+TOTAL_URLS=\${#URLS[@]}
+COMPLETED=0
+FAILED=0
+BATCH_SIZE=\${1:-$DEFAULT_BATCH_SIZE}
+if [ "$BATCH_SIZE" -eq "$DEFAULT_BATCH_SIZE" ] && [ -z "$1" ]; then
+    echo "note: the BATCH_SIZE can be set with a numerical argument after the command. e.g. bash this_script.txt 10"
+fi
+
+echo "Beginning parallel download of $TOTAL_URLS files (batch size: $BATCH_SIZE)"
+for ((i=0; i<TOTAL_URLS; i+=BATCH_SIZE)); do
+    pids=()
+    for ((j=i; j<i+BATCH_SIZE && j<TOTAL_URLS; j++)); do
+        download_file "\${URLS[j]}" &
+        pids+=($!)
+    done
+    
+    for pid in "\${pids[@]}"; do
+        wait $pid
+        status=$?
+        if [ $status -eq 0 ]; then
+            ((COMPLETED++))
+        else
+            ((FAILED++))
+        fi
+    done
+done
+
+echo ""
+if [ $FAILED -eq 0 ]; then
+    echo "Successfully downloaded $TOTAL_URLS files"
+else
+    echo "$FAILED files failed to download"
+fi
+echo ""
+echo "Press any key to exit..."
+read -n 1
+
+exit $FAILED
+`;

test/download_helper.js

@@ -31,16 +31,27 @@ describe("Download Helper", () => {
   });
 
   describe("generateDownloadList()", () => {
-    it("should create a string of curl calls with urls from a.item-button", async () => {
+    it("should create a URL array with urls from a.item-button", async () => {
       createDomNodes(`
         <span id="testId" class="download-title">
           <a class="item-button" href="url1"></button>
           <a class="item-button" href="url2"></button>
           <a class="item-button" href="url3"></button>
         </span>
       `);
-      const curlCommand =
-        'curl -OJ "url1" && \\\ncurl -OJ "url2" && \\\ncurl -OJ "url3"\n';
+      const curlCommand = `URLS=(\n\t"url1"\n\t"url2"\n\t"url3"\n)\n`;
+
+      const downloadList = DownloadHelper.generateDownloadList();
+      expect(downloadList).to.equal(curlCommand);
+    });
+
+    it("should create an empty URL array when no urls from a.item-button", async () => {
+      createDomNodes(`
+        <span id="testId" class="download-title">
+        </span>
+      `);
+      const curlCommand = `URLS=()\n`;
+
       const downloadList = DownloadHelper.generateDownloadList();
       expect(downloadList).to.equal(curlCommand);
     });
@@ -53,28 +64,28 @@ describe("Download Helper", () => {
       `);
 
       dh.createButton();
-      const button = document.querySelector(".downloadall");
+      const button = document.querySelector(".bes-downloadall");
 
       expect(button != null).to.be.true;
     });
 
     it("should not replace existing button at the end of div.download-titles", async () => {
       createDomNodes(`
         <div id="testId" class="download-titles">
-          <button class="downloadall fake" id="someId"></button>
+          <button class="bes-downloadall fake" id="someId"></button>
         </div>
       `);
 
       {
-        const button = document.querySelector(".downloadall");
+        const button = document.querySelector(".bes-downloadall");
         const idAttribute = button.getAttribute("id");
         expect(idAttribute).to.be.equal("someId");
       }
 
       dh.createButton();
 
       {
-        const button = document.querySelectorAll(".downloadall");
+        const button = document.querySelectorAll(".bes-downloadall");
         assert(button.length == 1);
         const idAttribute = button[0].getAttribute("id");
         expect(idAttribute).to.be.equal("someId");
@@ -88,7 +99,7 @@ describe("Download Helper", () => {
 
       dh.createButton();
 
-      const buttonClassName = "downloadall";
+      const buttonClassName = "bes-downloadall";
       expect(dh.button.className).to.equal(buttonClassName);
 
       const buttonTitleExpected =
@@ -106,6 +117,7 @@ describe("Download Helper", () => {
       DownloadHelper.dateString = sinon.fake.returns("dateString");
       DownloadHelper.generateDownloadList = sinon.fake.returns("downloadList");
       DownloadHelper.getDownloadPreamble = sinon.fake.returns("preamble");
+      DownloadHelper.getDownloadPostamble = sinon.fake.returns("postamble");
       DownloadHelper.downloadFile = sinon.spy();
 
       dh.button = document.createElement("button");
@@ -123,7 +135,7 @@ describe("Download Helper", () => {
         "bandcamp_dateString.txt"
       );
       expect(DownloadHelper.downloadFile.getCall(0).args[1]).to.equal(
-        "preamble" + "downloadList"
+        "preamble" + "downloadList" + "postamble"
       );
     });