scripts for managing the archive and mirror machines

Author: luizirber

machine/archive/README.md

@@ -0,0 +1,39 @@
+# Archive
+
+This machine contains a local copy of all wort signatures,
+synced from AWS S3 buckets.
+
+```
+.
+├── branchwater-index
+├── filelists
+│   ├── genomes_20250829
+│   ├── img_20250829
+│   └── sra_20250829
+├── manifests
+│   ├── genomes_20250829.csv
+│   ├── img_20250829.csv
+│   └── sra_20250829.csv
+├── s3_sync.sh
+├── scripts
+│   ├── csv_to_parquet.py
+│   ├── genomes.sbatch
+│   ├── img.sbatch
+│   └── sra.sbatch
+├── slurm_logs/
+├── update_logs/
+│   ├── 20250825_genomes
+│   ├── 20250825_img
+│   └── 20250825_sra
+├── wort-genomes
+│   └── sigs
+├── wort-img
+│   └── sigs
+└── wort-sra
+    └── sigs
+```
+
+## The `s3_sync.sh` script
+
+## Updating manifests
+

machine/archive/s3_sync.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -ex
+
+mkdir -p update_logs/
+
+# using s5cmd
+pixi exec s5cmd --stat --json sync --size-only 's3://wort-sra/*' wort-sra/ &> update_logs/"$(date +%Y%m%d)_sra" &
+pixi exec s5cmd --stat --json sync --size-only 's3://wort-genomes/*' wort-genomes/ &> update_logs/"$(date +%Y%m%d)_genomes" &
+pixi exec s5cmd --stat --json sync --size-only 's3://wort-img/*' wort-img/ &> update_logs/"$(date +%Y%m%d)_img" &
+wait

machine/archive/scripts/augment_csv.py

@@ -0,0 +1,51 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "polars",
+# ]
+# ///
+
+import polars as pl
+
+
+def main(args):
+    manifest_df = pl.scan_csv(args.manifest, skip_rows=1)
+    sha256_df = pl.scan_csv(
+        args.sha256,
+        has_header=False,
+        separator=" ",
+        new_columns=["sha256", "sep", "path"],
+    ).drop("sep")
+    df = manifest_df.join(sha256_df, left_on="internal_location", right_on="path")
+    if args.basepath:
+        df = df.with_columns(
+            pl.col("internal_location").str.strip_prefix(args.basepath)
+        )
+
+    match args.format:
+        case "parquet":
+            df.sink_parquet(
+                args.output,
+                compression_level=22,
+                metadata={"SOURMASH-MANIFEST-VERSION": "1.0"},
+            )
+        case "csv":
+            args.output.writelines(["# SOURMASH-MANIFEST-VERSION: 1.0\n"])
+            df.sink_csv(args.output)
+        case _:
+            print("Unknown output format")
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-b", "--basepath", default=None)
+    parser.add_argument("-F", "--format", choices=["parquet", "csv"], default="parquet")
+    parser.add_argument("manifest")
+    parser.add_argument("sha256")
+    parser.add_argument("output", type=argparse.FileType("w", encoding="utf-8"))
+
+    args = parser.parse_args()
+
+    main(args)

machine/archive/scripts/genomes.sbatch

@@ -0,0 +1,16 @@
+#!/bin/bash 
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=16
+#SBATCH --time=40:00:00
+#SBATCH --mem=20GB
+#SBATCH --partition=high2
+#SBATCH -A ctbrowngrp
+#SBATCH -o slurm_logs/genomes-%j.log
+
+set -exu -o pipefail
+
+cd $SLURM_SUBMIT_DIR
+
+#find wort-genomes/ -iname "*.sig" > filelists/genomes_20250829
+
+RAYON_NUM_THREADS=16 RUST_LOG=info ./branchwater-index manifest -o manifests/genomes_20250829.csv filelists/genomes_20250829

machine/archive/scripts/img.sbatch

@@ -0,0 +1,15 @@
+#!/bin/bash 
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=32
+#SBATCH --time=01:00:00
+#SBATCH --mem=10GB
+#SBATCH --partition=bmm
+#SBATCH -o slurm_logs/img-%j.log
+
+set -exu -o pipefail
+
+cd $SLURM_SUBMIT_DIR
+
+#find wort-img/ -iname "*.sig" > filelists/img_20250829
+
+RAYON_NUM_THREADS=32 RUST_LOG=info ./branchwater-index manifest -o manifests/img_20250829.csv filelists/img_20250829

machine/archive/scripts/sha256.sbatch

@@ -0,0 +1,16 @@
+#!/bin/bash 
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=16
+#SBATCH --time=40:00:00
+#SBATCH --mem=20GB
+#SBATCH --partition=high2
+#SBATCH -A ctbrowngrp
+#SBATCH -o slurm_logs/sha256-%j.log
+
+set -exu -o pipefail
+
+cd $SLURM_SUBMIT_DIR
+
+parallel -j16  sha256sum :::: filelists/img_20250829 > filelists/img_20250829.sha256
+parallel -j16  sha256sum :::: filelists/genomes_20250829 > filelists/genomes_20250829.sha256
+parallel -j16  sha256sum :::: filelists/sra_20250829 > filelists/sra_20250829.sha256

machine/archive/scripts/sra.sbatch

@@ -0,0 +1,16 @@
+#!/bin/bash 
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=32
+#SBATCH --time=100:00:00
+#SBATCH --mem=200GB
+#SBATCH --partition=high2
+#SBATCH -A ctbrowngrp
+#SBATCH -o slurm_logs/sra-%j.log
+
+set -exu -o pipefail
+
+cd $SLURM_SUBMIT_DIR
+
+#find wort-sra/ -iname "*.sig" > filelists/sra_20250829
+
+RAYON_NUM_THREADS=32 RUST_LOG=info ./branchwater-index manifest -o manifests/sra_20250829.csv filelists/sra_20250829