From 58e3c2c56ba9913bdcf883bc246b0d9d521ac1ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Valor=20Mart=C3=ADnez?= Date: Sun, 17 Jan 2021 11:57:30 +0100 Subject: [PATCH] generate_all --- download.sh | 52 ++----------------------------- generate.sh | 83 +++++++++++++++++++++++++++++++++++++++++++++++++ generate_all.sh | 8 +++++ 3 files changed, 94 insertions(+), 49 deletions(-) create mode 100755 generate.sh create mode 100755 generate_all.sh diff --git a/download.sh b/download.sh index ceddbaa..3280e8a 100755 --- a/download.sh +++ b/download.sh @@ -31,16 +31,13 @@ fi # variables readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -readonly DATA_DIR="$SCRIPT_DIR/data" readonly GAME_LIST_DIR="$SCRIPT_DIR/webdata" readonly SYSTEM_NAME="$1" readonly SYSTEM_HTML="$GAME_LIST_DIR/$SYSTEM_NAME".html readonly URL_LIST="$SCRIPT_DIR/${SYSTEM_NAME}.url" readonly COOKIE_FILE="$SCRIPT_DIR/cookie.txt" -readonly HASCHEEVOS_TXT="${DATA_DIR}/$1_hascheevos.txt" -readonly HASHLIBRARY_JSON="${DATA_DIR}/$1_hashlibrary.json" -readonly BLUE_COLOR=$( tput setaf 4 ) ; -readonly RESET_COLOR=$( tput sgr0 ) ; +readonly BLUE_COLOR=$(tput setaf 4) +readonly RESET_COLOR=$(tput sgr0) case $SYSTEM_NAME in @@ -178,7 +175,7 @@ wget "$URL" -O "$SYSTEM_HTML" # process system html to get game url list printf "%s\n" "${BLUE_COLOR}Processing $SYSTEM_NAME. ${RESET_COLOR}" -grep -E "Game/[0-9]+" -Eo "$SYSTEM_HTML" | sort -u | sed 's,Game/,https://retroachievements.org/linkedhashes.php?g=,' > "$URL_LIST" +grep -E "Game/[0-9]+" -Eo "$SYSTEM_HTML" | sort -u | sed 's,Game/,https://retroachievements.org/linkedhashes.php?g=,' >"$URL_LIST" printf "%s\n" "${BLUE_COLOR}Found $(wc -l <"$URL_LIST") games.${RESET_COLOR}" # create system folder for webpages with hashes @@ -199,46 +196,3 @@ wget --random-wait --no-verbose --load-cookies "${COOKIE_FILE}" -i "${URL_LIST}" if test -f "$URL_LIST"; then rm "$URL_LIST" fi - -# extract hashes from files on data/system folder - -echo "Processing $SYSTEM_NAME system." -echo "Working folder: $GAME_LIST_DIR" - -# if file exists, delete it -if test -f "${HASCHEEVOS_TXT}"; then - rm "${HASCHEEVOS_TXT}" -fi - -# if file exists, delete it -if test -f "${HASHLIBRARY_JSON}"; then - rm "${HASHLIBRARY_JSON}" -fi - -# check if folder exists -if ! test -d "${GAME_LIST_DIR}/${SYSTEM_NAME}"; then - echo "Folder ${GAME_LIST_DIR}/${SYSTEM_NAME} does not exist." - exit 0 -fi - -# create HASCHEEVOS_TXT file -grep -E "title=\"(.*?)\"" -Eo "${GAME_LIST_DIR}/${SYSTEM_NAME}/"* >> "${HASCHEEVOS_TXT}" -sed -i -E 's,^\/(.*?)g=,,g' "${HASCHEEVOS_TXT}" -sed -i -E 's,title=,true:,g' "${HASCHEEVOS_TXT}" -echo "File $HASCHEEVOS_TXT created." - -# create HASHLIBRARY_JSON -echo "{" > "${HASHLIBRARY_JSON}" -echo " \"Success\": true," >> "${HASHLIBRARY_JSON}" -echo " \"MD5List\": {" >> "${HASHLIBRARY_JSON}" -for i in "${GAME_LIST_DIR}/${SYSTEM_NAME}/"*; do - # catch hash and append to file - printf "%s\n" "$(grep -E "[A-Fa-f0-9]{32}" -Ho "$i")" >> "${HASHLIBRARY_JSON}" - printf "%s\r" "${BLUE_COLOR}Parsing $(basename "$i") ... ${RESET_COLOR}" -done -perl -i -pe 's/(^\/.*?=)([0-9]+):([0-9A-Za-z]{32})/ "$3": $2,/' "${HASHLIBRARY_JSON}" -echo " }" >> "${HASHLIBRARY_JSON}" -echo "}" >> "${HASHLIBRARY_JSON}" -perl -i -00pe 's/,(?!.*,)//s' "${HASHLIBRARY_JSON}" -printf "%s\n" "${BLUE_COLOR}Parsed $(wc -l < "${HASHLIBRARY_JSON}") hashes from $(find "${GAME_LIST_DIR}"/"$1" -type f | wc -l) games. ${RESET_COLOR}" -echo "File: $HASHLIBRARY_JSON created." diff --git a/generate.sh b/generate.sh new file mode 100755 index 0000000..0df1103 --- /dev/null +++ b/generate.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +readonly USAGE=" +USAGE: +$(basename "$0") [SYSTEM]" + +function help_message() { + echo "$USAGE" + echo + echo "Where [SYSTEM] are:" + for WORD in $SYSTEMS; do + echo -e "\t$WORD" + done +} + +SYSTEMS="apple2 arcade atari2600 atari7800 atarilynx coleco gamegear gb gba gbc mastersystem megadrive msx n64 nes ngpc pcengine sega32x sg1000 snes vectrex virtualboy wonderswan wonderswancolor" + +# check if there is any parameter +if [ "$#" -ne 1 ]; then + echo "You must enter a system name." + help_message + exit 0 +fi + +# check if the parameter is valid +if ! echo "$SYSTEMS" | grep -w "$1" >/dev/null; then + echo "You must enter a valid system name." + help_message + exit 0 +fi + +# variables +readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +readonly DATA_DIR="$SCRIPT_DIR/data" +readonly GAME_LIST_DIR="$SCRIPT_DIR/webdata" +readonly SYSTEM_NAME="$1" +readonly HASCHEEVOS_TXT="${DATA_DIR}/$1_hascheevos.txt" +readonly HASHLIBRARY_JSON="${DATA_DIR}/$1_hashlibrary.json" +readonly BLUE_COLOR=$(tput setaf 4) +readonly RESET_COLOR=$(tput sgr0) + +# extract hashes from files on data/system folder + +echo "Processing $SYSTEM_NAME system." +echo "Working folder: $GAME_LIST_DIR" + +# if file exists, delete it +if test -f "${HASCHEEVOS_TXT}"; then + rm "${HASCHEEVOS_TXT}" +fi + +# if file exists, delete it +if test -f "${HASHLIBRARY_JSON}"; then + rm "${HASHLIBRARY_JSON}" +fi + +# check if folder exists +if ! test -d "${GAME_LIST_DIR}/${SYSTEM_NAME}"; then + echo "Folder ${GAME_LIST_DIR}/${SYSTEM_NAME} does not exist." + exit 0 +fi + +# create HASCHEEVOS_TXT file +grep -E "title=\"(.*?)\"" -Eo "${GAME_LIST_DIR}/${SYSTEM_NAME}/"* >>"${HASCHEEVOS_TXT}" +sed -i -E 's,^\/(.*?)g=,,g' "${HASCHEEVOS_TXT}" +sed -i -E 's,title=,true:,g' "${HASCHEEVOS_TXT}" +echo "File $HASCHEEVOS_TXT created." + +# create HASHLIBRARY_JSON +echo "{" >"${HASHLIBRARY_JSON}" +echo " \"Success\": true," >>"${HASHLIBRARY_JSON}" +echo " \"MD5List\": {" >>"${HASHLIBRARY_JSON}" +for i in "${GAME_LIST_DIR}/${SYSTEM_NAME}/"*; do + # catch hash and append to file + printf "%s\n" "$(grep -E "[A-Fa-f0-9]{32}" -Ho "$i")" >>"${HASHLIBRARY_JSON}" + printf "%s\r" "${BLUE_COLOR}Parsing $(basename "$i") ... ${RESET_COLOR}" +done +perl -i -pe 's/(^\/.*?=)([0-9]+):([0-9A-Za-z]{32})/ "$3": $2,/' "${HASHLIBRARY_JSON}" +echo " }" >>"${HASHLIBRARY_JSON}" +echo "}" >>"${HASHLIBRARY_JSON}" +perl -i -00pe 's/,(?!.*,)//s' "${HASHLIBRARY_JSON}" +printf "%s\n" "${BLUE_COLOR}Parsed $(wc -l <"${HASHLIBRARY_JSON}") hashes from $(find "${GAME_LIST_DIR}"/"$1" -type f | wc -l) games. ${RESET_COLOR}" +echo "File: $HASHLIBRARY_JSON created." \ No newline at end of file diff --git a/generate_all.sh b/generate_all.sh new file mode 100755 index 0000000..52af2ed --- /dev/null +++ b/generate_all.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +readonly SYSTEMS="apple2 arcade atari2600 atari7800 atarilynx coleco gamegear gb gba gbc mastersystem megadrive msx n64 nes ngpc pcengine sega32x sg1000 snes vectrex virtualboy wonderswan wonderswancolor" + +for SYSTEM in $SYSTEMS; do + "${SCRIPT_DIR}"/generate.sh "${SYSTEM}" +done