JuiceFS is an open-source, high-performance distributed file system designed specifically for cloud environments. It offers unique features, such as:

• Separation of Data and Metadata: JuiceFS stores files in chunks within object storage like Google Cloud Storage, while metadata can be stored in various databases, including Redis.
• Performance: Achieves millisecond-level latency and nearly unlimited throughput, depending on the object storage scale.
• Easy Integration with MMCloud: MMCloud provides pre-configured nextflow head node templates with JuiceFS setup, simplifying deployment.
• Comparison with S3FS: For a detailed comparison between JuiceFS and S3FS, see JuiceFS vs. S3FS. JuiceFS typically offers better performance and scalability.

#!/bin/bash

# ---- User Configuration Section ----
# These configurations must be set by the user before running the script.

# ---- Optional Configuration Section ----
# These configurations are optional and can be customized as needed.

# JFS (JuiceFS) Private IP: Retrieved from the WORKER_ADDR environment variable.
jfs_private_ip=$(echo $WORKER_ADDR)

# Work Directory: Defines the root directory for working files. Optional suffix can be added.
workDir_suffix=''
workDir='/mnt/jfs/'$workDir_suffix
mkdir -p $workDir  # Ensures the working directory exists.
cd $workDir  # Changes to the working directory.
export NXF_HOME=$workDir  # Sets the NXF_HOME environment variable to the working directory.

# ---- Nextflow Configuration File Creation ----
# This section creates a Nextflow configuration file with various settings for the pipeline execution.

# Use cat to create or overwrite the mmc.config file with the desired Nextflow configurations.
# NOTE: GCS keys and OpCenter information will be concatted to the end of the config file. No need to add them now
cat > mmc.config << EOF
// enable nf-float plugin.
plugins {
    id 'nf-float'
}

// Process settings: Executor, error strategy, and resource allocation specifics.
process {
    executor = 'float'
    errorStrategy = 'retry'

    extra = '--dataVolume [opts=" --cache-dir /mnt/jfs_cache "]jfs://${jfs_private_ip}:6868/1:/mnt/jfs --dataVolume [size=120]:/mnt/jfs_cache --vmPolicy [spotOnly=true,retryLimit=10,retryInterval=300s]'
}

// Directories for Nextflow execution.
workDir = '${workDir}'
launchDir = '${workDir}'

EOF

# ---- Data Preparation ----
# Use this section to copy essential files from GCS to the working directory.

# For example, copy the sample sheet and params.yml from GCS to the current working directory.
# gcloud storage gs://nextflow-input/samplesheet.csv .
# gcloud storage gs://nextflow-input/scripts/params.yml .

# ---- Nextflow Command Setup ----
# Important: The -c option appends the mmc config file and soft overrides the nextflow configuration.

# Assembles the Nextflow command with all necessary options and parameters.
# The gs path needs to contain at least a sub-directory
# Bucket name MUST NOT contain underscores due to an internal GCS limitation
nextflow_command='nextflow run <nextflow-pipeline> \
-r <revision-number> \
-c mmc.config \
-params-file params.yml \
--input samplesheet.csv \
--outdir 'gs://nextflow-output/rnaseq/' \
-resume '

# -------------------------------------
# ---- DO NOT EDIT BELOW THIS LINE ----
# -------------------------------------
# The following section contains functions and commands that should not be modified by the user.

function install_float {
  # Install float
  local address=$(echo "$FLOAT_ADDR" | cut -d':' -f1)
  wget https://$address/float --no-check-certificate --quiet
  chmod +x float
}

function get_secret {
  input_string=$1
  local address=$(echo "$FLOAT_ADDR" | cut -d':' -f1)
  secret_value=$(./float secret get $input_string -a $address)
  if [[ $? -eq 0 ]]; then
    # Have this secret, will use the secret value
    echo $secret_value
    return
  else
    # Don't have this secret, will still use the input string
    echo $1
  fi
}

function gcloud_init() {
  # Login to gcloud with credential key
  gcloud auth login --quiet --cred-file=/mnt/jfs/gcp_cred.json
  if [[ $? -ne 0 ]]; then
    echo $(date): "gcloud authentication failed. Cannot dump JuiceFS metadata."
    exit 1
  fi
}

function remove_old_metadata () {
  echo $(date): "First finding and removing old metadata..."
  FOUND_METADATA=$(gcloud storage ls $BUCKET | grep ".meta.json.gz" |awk '{print $1}' | sed 's/gs:\/\///' | cut -d'/' -f2-)

  if [[ -z "$FOUND_METADATA" ]]; then
    echo $(date): "No previous metadata dump found. Continuing with dumping current JuiceFs"
  else
    echo $(date): "Previous metadata dump found! Removing $FOUND_METADATA"
    gcloud storage rm $BUCKET/$FOUND_METADATA

    echo $(date): "Previous metadata $FOUND_METADATA removed"
  fi

}

function dump_and_cp_metadata() {
  echo $(date): "Attempting to dump JuiceFS data"

  if [[ -z "$FOUND_METADATA" ]]; then
    # If no previous metadata was found, use the current job id
    juicefs dump redis://$(echo $WORKER_ADDR):6868/1 $(echo $FLOAT_JOB_ID).meta.json.gz --keep-secret-key
    echo $(date): "JuiceFS metadata $FLOAT_JOB_ID.meta.json.gz created. Copying to JuiceFS Bucket"
    gcloud storage cp "$(echo $FLOAT_JOB_ID).meta.json.gz" $BUCKET
  else
    # If previous metadata was found, use the id of the previous metadata
    metadata_name=$(echo $FOUND_METADATA | awk -F '.meta.json.gz' '{print $1}')
    juicefs dump redis://$(echo $WORKER_ADDR):6868/1 $(echo $metadata_name).meta.json.gz --keep-secret-key
    echo $(date): "JuiceFS metadata $metadata_name).meta.json.gz created. Copying to JuiceFS Bucket"
    gcloud storage cp "$(echo $metadata_name).meta.json.gz" $BUCKET
  fi

  echo $(date): "Copying to JuiceFS Bucket complete! Exiting..."
}

# Variables
FOUND_METADATA=""
export GOOGLE_APPLICATION_CREDENTIALS="/mnt/jfs/gcp_cred.json"

# Set Opcenter credentials
install_float
opcenter_ip_address=$(get_secret OPCENTER_IP_ADDRESS)
opcenter_username=$(get_secret OPCENTER_USERNAME)
opcenter_password=$(get_secret OPCENTER_PASSWORD)

# Append to config file
cat <<EOT >> mmc.config

// OpCenter connection settings.
float {
    address = '${opcenter_ip_address}'
    username = '${opcenter_username}'
    password = '${opcenter_password}'
}
EOT

# Initialize gcloud login
gcloud_init

# Start Nextflow run
$nextflow_command

# Error handling - JuiceFS dump and copy to gs bucket
if [[ $? -ne 0 ]]; then
  echo $(date): "Nextflow command failed."
  remove_old_metadata
  dump_and_cp_metadata
  exit 1
else 
  echo $(date): "Nextflow command succeeded."
  remove_old_metadata
  dump_and_cp_metadata
  exit 0
fi

# Create samplesheet
cat > samplesheet.csv << EOF
sample,fastq_1,fastq_2
SAMPLE1_PE,gs://mmc-gcp-poc/viralrecon-test-data/sample1_R1.fastq.gz,gs://mmc-gcp-poc/viralrecon-test-data/sample1_R2.fastq.gz
SAMPLE2_PE,gs://mmc-gcp-poc/viralrecon-test-data/sample2_R1.fastq.gz,gs://mmc-gcp-poc/viralrecon-test-data/sample2_R2.fastq.gz
SAMPLE3_SE,gs://mmc-gcp-poc/viralrecon-test-data/sample1_R1.fastq.gz,
SAMPLE3_SE,gs://mmc-gcp-poc/viralrecon-test-data/sample2_R1.fastq.gz,
EOF

# Create params file
cat > params.yml << EOF
multiqc_title: "rnaseq_multiqc"
fasta: "gs://nextflow-input/reference/Caenorhabditis_elegans.WBcel235.dna.toplevel.fa.gz"
gtf: "gs://nextflow-input/reference/Caenorhabditis_elegans.WBcel235.111.gtf.gz"
save_reference: true
remove_ribo_rna: true
skip_alignment: true
pseudo_aligner: "salmon"
EOF