diff --git a/2_download_system_game_webpages.sh b/2_download_system_game_webpages.sh index 7197bdc..0eec7ba 100755 --- a/2_download_system_game_webpages.sh +++ b/2_download_system_game_webpages.sh @@ -30,11 +30,16 @@ fi # variables readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +readonly DATA_DIR="$SCRIPT_DIR/data" readonly GAME_LIST_DIR="$SCRIPT_DIR/gamelists" readonly SYSTEM_NAME="$1" readonly SYSTEM_HTML="$GAME_LIST_DIR/$SYSTEM_NAME".html readonly URL_LIST="$SCRIPT_DIR/${SYSTEM_NAME}.url" readonly COOKIE_FILE="$SCRIPT_DIR/cookie.txt" +readonly HASCHEEVOS_TXT="${DATA_DIR}/$1_hascheevos.txt" +readonly HASHLIBRARY_JSON="${DATA_DIR}/$1_hashlibrary.json" +readonly BLUE_COLOR=$( tput setaf 4 ) ; +readonly RESET_COLOR=$( tput sgr0 ) ; case $SYSTEM_NAME in @@ -170,12 +175,12 @@ esac wget "$URL" -O "$SYSTEM_HTML" -#process system html to get game url list +# process system html to get game url list echo "Processing $SYSTEM_NAME." grep -E "Game/[0-9]+" -Eo "$SYSTEM_HTML" | sort -u | sed 's,Game/,https://retroachievements.org/linkedhashes.php?g=,' > "$URL_LIST" echo "Found $(wc -l <"$URL_LIST") games." -#create system folder for webpages with hashes +# create system folder for webpages with hashes if test -d "./${GAME_LIST_DIR}/$SYSTEM_NAME"; then rm -rdf "./${GAME_LIST_DIR}/$SYSTEM_NAME" echo "Deleting ${GAME_LIST_DIR}/$SYSTEM_NAME folder." @@ -185,11 +190,54 @@ if ! test -d "./${GAME_LIST_DIR}/$SYSTEM_NAME"; then echo "Create ${GAME_LIST_DIR}/$SYSTEM_NAME folder." fi -#download webpages with hashes form url list +# download webpages with hashes form url list echo "Downloading game info ..." wget --random-wait --no-verbose --load-cookies "${COOKIE_FILE}" -i "${URL_LIST}" -P "./${GAME_LIST_DIR}/${SYSTEM_NAME}" -#remove tempfiles +# remove tempfiles if test -f "$URL_LIST"; then rm "$URL_LIST" fi + +# extract hashes from files on data/system folder + +echo "Processing $SYSTEM_NAME system." +echo "Working folder: $GAME_LIST_DIR" + +# if file exists, delete it +if test -f "${HASCHEEVOS_TXT}"; then + rm "${HASCHEEVOS_TXT}" +fi + +# if file exists, delete it +if test -f "${HASHLIBRARY_JSON}"; then + rm "${HASHLIBRARY_JSON}" +fi + +# check if folder exists +if ! test -d "${GAME_LIST_DIR}/${SYSTEM_NAME}"; then + echo "Folder ${GAME_LIST_DIR}/${SYSTEM_NAME} does not exist." + exit 0 +fi + +# create HASCHEEVOS_TXT file +grep -E "title=\"(.*?)\"" -Eo "${GAME_LIST_DIR}/${SYSTEM_NAME}/"* >> "${HASCHEEVOS_TXT}" +sed -i -E 's,^\/(.*?)g=,,g' "${HASCHEEVOS_TXT}" +sed -i -E 's,title=,true:,g' "${HASCHEEVOS_TXT}" +echo "File $HASCHEEVOS_TXT created." + +# create HASHLIBRARY_JSON +echo "{" > "${HASHLIBRARY_JSON}" +echo " \"Success\": true," >> "${HASHLIBRARY_JSON}" +echo " \"MD5List\": {" >> "${HASHLIBRARY_JSON}" +for i in "${GAME_LIST_DIR}/${SYSTEM_NAME}/"*; do + # catch hash and append to file + printf "%s\n" "$(grep -E "[A-Fa-f0-9]{32}" -Ho "$i")" >> "${HASHLIBRARY_JSON}" + printf "%s\r" "${BLUE_COLOR}Parsing $(basename "$i") ... ${RESET_COLOR}" +done +perl -i -pe 's/(^\/.*?=)([0-9]+):([0-9A-Za-z]{32})/ "$3": $2,/' "${HASHLIBRARY_JSON}" +echo " }" >> "${HASHLIBRARY_JSON}" +echo "}" >> "${HASHLIBRARY_JSON}" +perl -i -00pe 's/,(?!.*,)//s' "${HASHLIBRARY_JSON}" +printf "%s\n" "Parsed $(wc -l < "${HASHLIBRARY_JSON}") hashes from $(find "${GAME_LIST_DIR}"/"$1" -type f | wc -l) games. " +echo "File: $HASHLIBRARY_JSON created." \ No newline at end of file