You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
35 lines
767 B
35 lines
767 B
#!/bin/bash
|
|
|
|
model=$1
|
|
count=${2:-9000}
|
|
|
|
mkdir -p ${model}/downloads
|
|
rm -rf ${model}/downloads/*
|
|
|
|
BOOK_IDS=(
|
|
174 1342 11 2701 84 2600 1661 215 1952 98 345 35 1497 2147 4300
|
|
)
|
|
|
|
echo "Downloading books..."
|
|
|
|
i=1
|
|
for ID in "${BOOK_IDS[@]}"; do
|
|
echo "Fetching $ID..."
|
|
curl -s -L "https://www.gutenberg.org/files/$ID/$ID-0.txt" -o "${model}/downloads/$ID.txt"
|
|
i=$((i+1))
|
|
if [ ${i} -gt ${count} ]; then
|
|
break;
|
|
fi
|
|
done
|
|
|
|
echo "Concatenating into corpus.txt..."
|
|
|
|
# Remove headers/footers and join them
|
|
rm -f ${model}/corpus.txt
|
|
for FILE in ${model}/downloads/*.txt; do
|
|
awk '/\*\*\* START OF/,/\*\*\* END OF/ { if (!/START|END/) print }' "$FILE" >> ${model}/corpus.txt
|
|
echo >> ${model}/corpus.txt
|
|
done
|
|
|
|
echo "All done. Corpus size:"
|
|
du -h ${model}/corpus.txt
|
|
|