diff --git a/1_download_system_game_list.sh b/1_download_system_game_list.sh index d03f30d..1dbff16 100755 --- a/1_download_system_game_list.sh +++ b/1_download_system_game_list.sh @@ -1,66 +1,66 @@ #!/bin/bash -apple2=https://retroachievements.org/gameList.php?c=38 -arcade=https://retroachievements.org/gameList.php?c=27 -atari2600=https://retroachievements.org/gameList.php?c=25 -atari7800=https://retroachievements.org/gameList.php?c=51 -atarijaguar=https://retroachievements.org/gameList.php?c=17 -atarilynx=https://retroachievements.org/gameList.php?c=13 -coleco=https://retroachievements.org/gameList.php?c=44 -gba=https://retroachievements.org/gameList.php?c=5 -gbc=https://retroachievements.org/gameList.php?c=6 -gb=https://retroachievements.org/gameList.php?c=4 -gamegear=https://retroachievements.org/gameList.php?c=15 -mastersystem=https://retroachievements.org/gameList.php?c=11 -megadrive=https://retroachievements.org/gameList.php?c=1 -msx=https://retroachievements.org/gameList.php?c=29 -ngpc=https://retroachievements.org/gameList.php?c=14 -nes=https://retroachievements.org/gameList.php?c=7 -n64=https://retroachievements.org/gameList.php?c=2 -nds=https://retroachievements.org/gameList.php?c=18 -panasonic3do=https://retroachievements.org/gameList.php?c=43 -pc8000=https://retroachievements.org/gameList.php?c=47 -pcengine=https://retroachievements.org/gameList.php?c=8 -pcfx=https://retroachievements.org/gameList.php?c=49 -psx=https://retroachievements.org/gameList.php?c=12 -pokemonmini=https://retroachievements.org/gameList.php?c=24 -sega32x=https://retroachievements.org/gameList.php?c=10 -segacd=https://retroachievements.org/gameList.php?c=9 -segasaturn=https://retroachievements.org/gameList.php?c=39 -sg1000=https://retroachievements.org/gameList.php?c=33 -snes=https://retroachievements.org/gameList.php?c=3 -vectrex=https://retroachievements.org/gameList.php?c=46 -virtualboy=https://retroachievements.org/gameList.php?c=28 -wonderswan=https://retroachievements.org/gameList.php?c=53 +apple2="https://retroachievements.org/gameList.php?c=38" +arcade="https://retroachievements.org/gameList.php?c=27" +atari2600="https://retroachievements.org/gameList.php?c=25" +atari7800="https://retroachievements.org/gameList.php?c=51" +atarijaguar="https://retroachievements.org/gameList.php?c=17" +atarilynx="https://retroachievements.org/gameList.php?c=13" +coleco="https://retroachievements.org/gameList.php?c=44" +gba="https://retroachievements.org/gameList.php?c=5" +gbc="https://retroachievements.org/gameList.php?c=6" +gb="https://retroachievements.org/gameList.php?c=4" +gamegear="https://retroachievements.org/gameList.php?c=15" +mastersystem="https://retroachievements.org/gameList.php?c=11" +megadrive="https://retroachievements.org/gameList.php?c=1" +msx="https://retroachievements.org/gameList.php?c=29" +ngpc="https://retroachievements.org/gameList.php?c=14" +nes="https://retroachievements.org/gameList.php?c=7" +n64="https://retroachievements.org/gameList.php?c=2" +nds="https://retroachievements.org/gameList.php?c=18" +panasonic3do="https://retroachievements.org/gameList.php?c=43" +pc8000="https://retroachievements.org/gameList.php?c=47" +pcengine="https://retroachievements.org/gameList.php?c=8" +pcfx="https://retroachievements.org/gameList.php?c=49" +psx="https://retroachievements.org/gameList.php?c=12" +pokemonmini="https://retroachievements.org/gameList.php?c=24" +sega32x="https://retroachievements.org/gameList.php?c=10" +segacd="https://retroachievements.org/gameList.php?c=9" +segasaturn="https://retroachievements.org/gameList.php?c=39" +sg1000="https://retroachievements.org/gameList.php?c=33" +snes="https://retroachievements.org/gameList.php?c=3" +vectrex="https://retroachievements.org/gameList.php?c=46" +virtualboy="https://retroachievements.org/gameList.php?c=28" +wonderswan="https://retroachievements.org/gameList.php?c=53" -wget $apple2 -O apple2.html -wget $arcade -O arcade.html -wget $atari2600 -O atari2600.html -wget $atari7800 -O atari7800.html -wget $atarijaguar -O atarijaguar.html -wget $atarilynx -O atarilynx.html -wget $coleco -O coleco.html -wget $gba -O gba.html -wget $gbc -O gbc.html -wget $gb -O gb.html -wget $gamegear -O gamegear.html -wget $mastersystem -O mastersystem.html -wget $megadrive -O megadrive.html -wget $msx -O msx.html -wget $ngpc -O ngpc.html -wget $nes -O nes.html -wget $n64 -O n64.html -wget $nds -O nds.html -wget $panasonic3do -O panasonic3do.html -wget $pc8000 -O pc8000.html -wget $pcengine -O pcengine.html -wget $pcfx -O pcfx.html -wget $psx -O psx.html -wget $pokemonmini -O pokemonmini.html -wget $sega32x -O sega32x.html -wget $segacd -O segacd.html -wget $segasaturn -O segasaturn.html -wget $sg1000 -O sg1000.html -wget $snes -O snes.html -wget $vectrex -O vectrex.html -wget $virtualboy -O virtualboy.html -wget $wonderswan -O wonderswan.html +wget "$apple2" -O apple2.html +wget "$arcade" -O arcade.html +wget "$atari2600" -O atari2600.html +wget "$atari7800" -O atari7800.html +wget "$atarijaguar" -O atarijaguar.html +wget "$atarilynx" -O atarilynx.html +wget "$coleco" -O coleco.html +wget "$gba" -O gba.html +wget "$gbc" -O gbc.html +wget "$gb" -O gb.html +wget "$gamegear" -O gamegear.html +wget "$mastersystem" -O mastersystem.html +wget "$megadrive" -O megadrive.html +wget "$msx" -O msx.html +wget "$ngpc" -O ngpc.html +wget "$nes" -O nes.html +wget "$n64" -O n64.html +wget "$nds" -O nds.html +wget "$panasonic3do" -O panasonic3do.html +wget "$pc8000" -O pc8000.html +wget "$pcengine" -O pcengine.html +wget "$pcfx" -O pcfx.html +wget "$psx" -O psx.html +wget "$pokemonmini" -O pokemonmini.html +wget "$sega32x" -O sega32x.html +wget "$segacd" -O segacd.html +wget "$segasaturn" -O segasaturn.html +wget "$sg1000" -O sg1000.html +wget "$snes" -O snes.html +wget "$vectrex" -O vectrex.html +wget "$virtualboy" -O virtualboy.html +wget "$wonderswan" -O wonderswan.html diff --git a/2_download_system_game_webpages.sh b/2_download_system_game_webpages.sh index d20347d..fdcbe68 100755 --- a/2_download_system_game_webpages.sh +++ b/2_download_system_game_webpages.sh @@ -1,15 +1,25 @@ #!/bin/bash - -readonly SCRIPT_DIR="$(cd "$(dirname $0)" && pwd)" -readonly DATA_DIR="$SCRIPT_DIR/data" -readonly GAME_LIST_DIR="$SCRIPT_DIR/gamelists" +SYSTEMS="apple2 arcade atari2600 atari7800 atarilynx coleco gamegear gb gba gbc mastersystem megadrive msx n64 nes ngpc pcengine sega32x sg1000 snes vectrex virtualboy wonderswan wonderswancolor" #check if there is any parameter if [ "$#" -ne 1 ]; then - echo "You must enter a source file." + echo "You must enter a system name." exit 0 fi +# check if the parameter is valid +if ! echo "$SYSTEMS" | grep -w "$1" > /dev/null; then + echo "You must enter a valid system name." + echo "Supported systemas are:" + for WORD in $SYSTEMS + do + echo -e "\t$WORD" + done + exit 0 +fi + + + #check if file exists if ! test -f "$1"; then echo "$1 does not exists." @@ -17,57 +27,32 @@ if ! test -f "$1"; then fi #variables -hashDir=data -gameListDir=gamelists -systemName="$(basename "$1" .html)" -echo "Processing $systemName." -tempFile1="${systemName}-1.tmp" -tempFile2="${systemName}-2.tmp" -urlList="${systemName}.url" -hashList="./${hashDir}/${systemName}.hashlist" -cookie="cookie.txt" - -#update cookie time stamp -#sec=`date +%s` -#sec2=$((sec+600)) -#sed -i 's/[0-9]\{10\}/'$sec2'/' cookie.txt +readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +readonly GAME_LIST_DIR="$SCRIPT_DIR/gamelists" +SYSTEM_NAME="$1" +URL_LIST="$SCRIPT_DIR/${SYSTEM_NAME}.url" +COOKIE_FILE="$SCRIPT_DIR/cookie.txt" #process system html to get game url list -grep -E "Game/[1-9]'" -Eo $1 > $tempFile1 -grep -E "Game/[1-9][0-9]'" -Eo $1 >> $tempFile1 -grep -E "Game/[1-9][0-9][0-9]'" -Eo $1 >> $tempFile1 -grep -E "Game/[1-9][0-9][0-9][0-9]'" -Eo $1 >> $tempFile1 -grep -E "Game/[1-9][0-9][0-9][0-9][0-9]'" -Eo $1 >> $tempFile1 -grep -E "Game/[1-9][0-9][0-9][0-9][0-9][0-9]'" -Eo $1 >> $tempFile1 - -sort -u $tempFile1 > $tempFile2 -sed 's,Game/,,' $tempFile2 > $tempFile1 -sed 's,^,https://retroachievements.org/linkedhashes.php?g=,' $tempFile1 > $tempFile2 -sed 's,.$,,' $tempFile2 > $urlList - -echo "Found $(wc -l < $urlList) games." +echo "Processing $SYSTEM_NAME." +grep -E "Game/[0-9]+" -Eo "$1" | sort -u | sed 's,Game/,https://retroachievements.org/linkedhashes.php?g=,' "$URL_LIST" +echo "Found $(wc -l < "$URL_LIST") games." #create system folder for webpages with hashes -if test -d "./${gameListDir}/$systemName"; then - rm -rdf "./${gameListDir}/$systemName" - echo "Deleting ${gameListDir}/$systemName folder." +if test -d "./${GAME_LIST_DIR}/$SYSTEM_NAME"; then + rm -rdf "./${GAME_LIST_DIR}/$SYSTEM_NAME" + echo "Deleting ${GAME_LIST_DIR}/$SYSTEM_NAME folder." fi -if ! test -d "./${gameListDir}/$systemName"; then - mkdir -p "./${gameListDir}/$systemName" - echo "Create ${gameListDir}/$systemName folder." +if ! test -d "./${GAME_LIST_DIR}/$SYSTEM_NAME"; then + mkdir -p "./${GAME_LIST_DIR}/$SYSTEM_NAME" + echo "Create ${GAME_LIST_DIR}/$SYSTEM_NAME folder." fi #download webpages with hashes form url list echo "Downloading game info ..." -wget --random-wait --no-verbose --load-cookies ${cookie} -i $urlList -P "./${gameListDir}/$systemName" +wget --random-wait --no-verbose --load-COOKIE_FILEs "${COOKIE_FILE}" -i "${URL_LIST}" -P "./${GAME_LIST_DIR}/${SYSTEM_NAME}" #remove tempfiles -if test -f "$tempFile1"; then - rm "$tempFile1" -fi -if test -f "$tempFile2"; then - rm "$tempFile2" -fi -if test -f "$urlList"; then - rm "$urlList" +if test -f "$URL_LIST"; then + rm "$URL_LIST" fi diff --git a/3_extract_hascheevos_hash_from_game_webpages.sh b/3_extract_hascheevos_hash_from_game_webpages.sh index 74c2c5e..8b8f90b 100755 --- a/3_extract_hascheevos_hash_from_game_webpages.sh +++ b/3_extract_hascheevos_hash_from_game_webpages.sh @@ -14,7 +14,7 @@ if [ "$#" -ne 1 ]; then fi # check if the parameter is valid -if ! echo $SYSTEMS | grep -w $1 > /dev/null; then +if ! echo "$SYSTEMS" | grep -w "$1" > /dev/null; then echo "You must enter a valid system name." echo "Supported systemas are:" for WORD in $SYSTEMS @@ -24,7 +24,7 @@ if ! echo $SYSTEMS | grep -w $1 > /dev/null; then exit 0 fi -readonly SCRIPT_DIR="$(cd "$(dirname $0)" && pwd)" +readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" readonly DATA_DIR="$SCRIPT_DIR/data" readonly GAME_LIST_DIR="$SCRIPT_DIR/gamelists" readonly HASCHEEVOS_TXT="${DATA_DIR}/$1_hascheevos.txt" @@ -54,23 +54,23 @@ if ! test -d "${GAME_LIST_DIR}/$1"; then fi # create HASCHEEVOS_TXT file -grep -E "title=\"(.*?)\"" -Eo "${GAME_LIST_DIR}/$1/"* >> ${HASCHEEVOS_TXT} -sed -i -E 's,^\/(.*?)g=,,g' ${HASCHEEVOS_TXT} -sed -i -E 's,title=,true:,g' ${HASCHEEVOS_TXT} +grep -E "title=\"(.*?)\"" -Eo "${GAME_LIST_DIR}/$1/"* >> "${HASCHEEVOS_TXT}" +sed -i -E 's,^\/(.*?)g=,,g' "${HASCHEEVOS_TXT}" +sed -i -E 's,title=,true:,g' "${HASCHEEVOS_TXT}" echo "File $HASCHEEVOS_TXT created." # create HASHLIBRARY_JSON -echo "{" > ${HASHLIBRARY_JSON} -echo " \"Success\": true," >> ${HASHLIBRARY_JSON} -echo " \"MD5List\": {" >> ${HASHLIBRARY_JSON} +echo "{" > "${HASHLIBRARY_JSON}" +echo " \"Success\": true," >> "${HASHLIBRARY_JSON}" +echo " \"MD5List\": {" >> "${HASHLIBRARY_JSON}" for i in "${GAME_LIST_DIR}/$1/"*; do # catch hash and append to file - printf "%s\n" "$(grep -E "[A-Fa-f0-9]{32}" -Ho $i)" >> ${HASHLIBRARY_JSON} - printf "%s\r" "${BLUE_COLOR}Parsing $(basename ${i}) ... ${RESET_COLOR}" + printf "%s\n" "$(grep -E "[A-Fa-f0-9]{32}" -Ho "$i")" >> "${HASHLIBRARY_JSON}" + printf "%s\r" "${BLUE_COLOR}Parsing $(basename "$i") ... ${RESET_COLOR}" done -perl -i -pe 's/(^\/.*?=)([0-9]+):([0-9A-Za-z]{32})/ "$3": $2,/' ${HASHLIBRARY_JSON} -echo " }" >> ${HASHLIBRARY_JSON} -echo "}" >> ${HASHLIBRARY_JSON} -perl -i -00pe 's/,(?!.*,)//s' ${HASHLIBRARY_JSON} -printf "%s\n" "Parsed $(wc -l < ${HASHLIBRARY_JSON}) hashes from $(find ${GAME_LIST_DIR}/$1 -type f | wc -l) games. " -echo "File: $HASHLIBRARY_JSON created." +perl -i -pe 's/(^\/.*?=)([0-9]+):([0-9A-Za-z]{32})/ "$3": $2,/' "${HASHLIBRARY_JSON}" +echo " }" >> "${HASHLIBRARY_JSON}" +echo "}" >> "${HASHLIBRARY_JSON}" +perl -i -00pe 's/,(?!.*,)//s' "${HASHLIBRARY_JSON}" +printf "%s\n" "Parsed $(wc -l < "${HASHLIBRARY_JSON}") hashes from $(find "${GAME_LIST_DIR}"/"$1" -type f | wc -l) games. " +echo "File: $HASHLIBRARY_JSON created." \ No newline at end of file