First commit

2_download_system_game_webpages.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+readonly SCRIPT_DIR="$(cd "$(dirname $0)" && pwd)"
+readonly DATA_DIR="$SCRIPT_DIR/data"
+readonly GAME_LIST_DIR="$SCRIPT_DIR/gamelists"
+
+#check if there is any parameter
+if [ "$#" -ne 1 ]; then
+    echo "You must enter a source file."
+    exit 0
+fi
+
+#check if file exists
+if ! test -f "$1"; then
+    echo "$1 does not exists."
+    exit 0
+fi
+
+#variables
+hashDir=data
+gameListDir=gamelists
+systemName="$(basename "$1" .html)"
+echo "Processing $systemName."
+tempFile1="${systemName}-1.tmp"
+tempFile2="${systemName}-2.tmp"
+urlList="${systemName}.url"
+hashList="./${hashDir}/${systemName}.hashlist"
+cookie="cookie.txt"
+
+#update cookie time stamp
+#sec=`date +%s`
+#sec2=$((sec+600))
+#sed -i 's/[0-9]\{10\}/'$sec2'/' cookie.txt
+
+#process system html to get game url list
+grep -E "Game/[1-9]'" -Eo $1 > $tempFile1
+grep -E "Game/[1-9][0-9]'" -Eo $1 >> $tempFile1
+grep -E "Game/[1-9][0-9][0-9]'" -Eo $1 >> $tempFile1
+grep -E "Game/[1-9][0-9][0-9][0-9]'" -Eo $1 >> $tempFile1
+grep -E "Game/[1-9][0-9][0-9][0-9][0-9]'" -Eo $1 >> $tempFile1
+grep -E "Game/[1-9][0-9][0-9][0-9][0-9][0-9]'" -Eo $1 >> $tempFile1
+
+sort -u $tempFile1 > $tempFile2
+sed 's,Game/,,' $tempFile2 > $tempFile1
+sed 's,^,https://retroachievements.org/linkedhashes.php?g=,' $tempFile1 > $tempFile2
+sed 's,.$,,' $tempFile2 > $urlList
+
+echo "Found $(wc -l < $urlList) games."
+
+#create system folder for webpages with hashes
+if test -d "./${gameListDir}/$systemName"; then
+    rm -rdf "./${gameListDir}/$systemName"
+    echo "Deleting ${gameListDir}/$systemName folder."
+fi
+if ! test -d "./${gameListDir}/$systemName"; then
+    mkdir -p "./${gameListDir}/$systemName"
+    echo "Create ${gameListDir}/$systemName folder."
+fi
+
+#download webpages with hashes form url list
+echo "Downloading game info ..."
+wget --random-wait --no-verbose --load-cookies ${cookie} -i $urlList -P "./${gameListDir}/$systemName"
+
+#remove tempfiles
+if test -f "$tempFile1"; then
+    rm "$tempFile1"
+fi
+if test -f "$tempFile2"; then
+    rm "$tempFile2"
+fi
+if test -f "$urlList"; then
+    rm "$urlList"
+fi