#!/bin/bash

model=$1
count=${2:-9000}

mkdir -p ${model}/downloads
rm -rf ${model}/downloads/*

BOOK_IDS=(
  174 1342 11 2701 84 2600 1661 215 1952 98 345 35 1497 2147 4300
)

echo "Downloading books..."

i=1
for ID in "${BOOK_IDS[@]}"; do
  echo "Fetching $ID..."
  curl -s -L "https://www.gutenberg.org/files/$ID/$ID-0.txt" -o "${model}/downloads/$ID.txt"
  i=$((i+1))
  if [ ${i} -gt ${count} ]; then
    break;
  fi
done

echo "Concatenating into corpus.txt..."

# Remove headers/footers and join them
rm -f ${model}/corpus.txt
for FILE in ${model}/downloads/*.txt; do
  awk '/\*\*\* START OF/,/\*\*\* END OF/ { if (!/START|END/) print }' "$FILE" >> ${model}/corpus.txt
  echo >> ${model}/corpus.txt
done

echo "All done. Corpus size:"
du -h ${model}/corpus.txt