merged script 2 with script 3

2_download_system_game_webpages.sh

@@ -30,11 +30,16 @@ fi
 
 # variables
 readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+readonly DATA_DIR="$SCRIPT_DIR/data"
 readonly GAME_LIST_DIR="$SCRIPT_DIR/gamelists"
 readonly SYSTEM_NAME="$1"
 readonly SYSTEM_HTML="$GAME_LIST_DIR/$SYSTEM_NAME".html
 readonly URL_LIST="$SCRIPT_DIR/${SYSTEM_NAME}.url"
 readonly COOKIE_FILE="$SCRIPT_DIR/cookie.txt"
+readonly HASCHEEVOS_TXT="${DATA_DIR}/$1_hascheevos.txt"
+readonly HASHLIBRARY_JSON="${DATA_DIR}/$1_hashlibrary.json"
+readonly BLUE_COLOR=$( tput setaf 4 ) ;
+readonly RESET_COLOR=$( tput sgr0 ) ;
 
 case $SYSTEM_NAME in
 
@@ -170,12 +175,12 @@ esac
 
 wget "$URL" -O "$SYSTEM_HTML"
 
-#process system html to get game url list
+# process system html to get game url list
 echo "Processing $SYSTEM_NAME."
 grep -E "Game/[0-9]+" -Eo "$SYSTEM_HTML" | sort -u | sed 's,Game/,https://retroachievements.org/linkedhashes.php?g=,' > "$URL_LIST"
 echo "Found $(wc -l <"$URL_LIST") games."
 
-#create system folder for webpages with hashes
+# create system folder for webpages with hashes
 if test -d "./${GAME_LIST_DIR}/$SYSTEM_NAME"; then
     rm -rdf "./${GAME_LIST_DIR}/$SYSTEM_NAME"
     echo "Deleting ${GAME_LIST_DIR}/$SYSTEM_NAME folder."
@@ -185,11 +190,54 @@ if ! test -d "./${GAME_LIST_DIR}/$SYSTEM_NAME"; then
     echo "Create ${GAME_LIST_DIR}/$SYSTEM_NAME folder."
 fi
 
-#download webpages with hashes form url list
+# download webpages with hashes form url list
 echo "Downloading game info ..."
 wget --random-wait --no-verbose --load-cookies "${COOKIE_FILE}" -i "${URL_LIST}" -P "./${GAME_LIST_DIR}/${SYSTEM_NAME}"
 
-#remove tempfiles
+# remove tempfiles
 if test -f "$URL_LIST"; then
     rm "$URL_LIST"
 fi
+
+# extract hashes from files on data/system folder
+
+echo "Processing $SYSTEM_NAME system."
+echo "Working folder: $GAME_LIST_DIR"
+
+# if file exists, delete it
+if test -f "${HASCHEEVOS_TXT}"; then
+    rm "${HASCHEEVOS_TXT}"
+fi
+
+# if file exists, delete it
+if test -f "${HASHLIBRARY_JSON}"; then
+    rm "${HASHLIBRARY_JSON}"
+fi
+
+# check if folder exists
+if ! test -d "${GAME_LIST_DIR}/${SYSTEM_NAME}"; then
+    echo "Folder ${GAME_LIST_DIR}/${SYSTEM_NAME} does not exist."
+    exit 0
+fi
+
+# create HASCHEEVOS_TXT file
+grep -E "title=\"(.*?)\"" -Eo "${GAME_LIST_DIR}/${SYSTEM_NAME}/"* >> "${HASCHEEVOS_TXT}"
+sed -i -E 's,^\/(.*?)g=,,g' "${HASCHEEVOS_TXT}"
+sed -i -E 's,title=,true:,g' "${HASCHEEVOS_TXT}"
+echo "File $HASCHEEVOS_TXT created."
+
+# create HASHLIBRARY_JSON
+echo "{" > "${HASHLIBRARY_JSON}"
+echo "  \"Success\": true," >> "${HASHLIBRARY_JSON}"
+echo "  \"MD5List\": {" >> "${HASHLIBRARY_JSON}"
+for i in "${GAME_LIST_DIR}/${SYSTEM_NAME}/"*; do
+    # catch hash and append to file
+    printf "%s\n" "$(grep -E "[A-Fa-f0-9]{32}" -Ho "$i")" >> "${HASHLIBRARY_JSON}"
+    printf "%s\r" "${BLUE_COLOR}Parsing $(basename "$i") ... ${RESET_COLOR}"
+done
+perl -i -pe 's/(^\/.*?=)([0-9]+):([0-9A-Za-z]{32})/    "$3": $2,/' "${HASHLIBRARY_JSON}"
+echo "  }" >> "${HASHLIBRARY_JSON}"
+echo "}" >> "${HASHLIBRARY_JSON}"
+perl -i -00pe 's/,(?!.*,)//s' "${HASHLIBRARY_JSON}"
+printf "%s\n" "Parsed $(wc -l < "${HASHLIBRARY_JSON}") hashes from $(find "${GAME_LIST_DIR}"/"$1" -type f | wc -l) games.                  "
+echo "File: $HASHLIBRARY_JSON created."