diff --git a/download.sh b/download.sh new file mode 100755 index 0000000..a99df08 --- /dev/null +++ b/download.sh @@ -0,0 +1,244 @@ +#!/bin/bash + +readonly USAGE=" +USAGE: +$(basename "$0") [SYSTEM]" + +function help_message() { + echo "$USAGE" + echo + echo "Where [SYSTEM] are:" + for WORD in $SYSTEMS; do + echo -e "\t$WORD" + done +} + +SYSTEMS="apple2 arcade atari2600 atari7800 atarilynx coleco gamegear gb gba gbc mastersystem megadrive msx n64 nes ngpc pcengine sega32x sg1000 snes vectrex virtualboy wonderswan wonderswancolor" + +# check if there is any parameter +if [ "$#" -ne 1 ]; then + echo "You must enter a system name." + help_message + exit 0 +fi + +# check if the parameter is valid +if ! echo "$SYSTEMS" | grep -w "$1" >/dev/null; then + echo "You must enter a valid system name." + help_message + exit 0 +fi + +# variables +readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +readonly DATA_DIR="$SCRIPT_DIR/data" +readonly GAME_LIST_DIR="$SCRIPT_DIR/webdata" +readonly SYSTEM_NAME="$1" +readonly SYSTEM_HTML="$GAME_LIST_DIR/$SYSTEM_NAME".html +readonly URL_LIST="$SCRIPT_DIR/${SYSTEM_NAME}.url" +readonly COOKIE_FILE="$SCRIPT_DIR/cookie.txt" +readonly HASCHEEVOS_TXT="${DATA_DIR}/$1_hascheevos.txt" +readonly HASHLIBRARY_JSON="${DATA_DIR}/$1_hashlibrary.json" +readonly BLUE_COLOR=$( tput setaf 4 ) ; +readonly RESET_COLOR=$( tput sgr0 ) ; + +case $SYSTEM_NAME in + +apple2) + readonly URL="https://retroachievements.org/gameList.php?c=38" + ;; + +arcade) + readonly URL="https://retroachievements.org/gameList.php?c=27" + ;; + +atari2600) + readonly URL="https://retroachievements.org/gameList.php?c=25" + ;; + +atari7800) + readonly URL="https://retroachievements.org/gameList.php?c=51" + ;; + +atarijaguar) + readonly URL="https://retroachievements.org/gameList.php?c=17" + ;; + +atarilynx) + readonly URL="https://retroachievements.org/gameList.php?c=13" + ;; + +coleco) + readonly URL="https://retroachievements.org/gameList.php?c=44" + ;; + +gba) + readonly URL="https://retroachievements.org/gameList.php?c=5" + ;; + +gbc) + readonly URL="https://retroachievements.org/gameList.php?c=6" + ;; + +gb) + readonly URL="https://retroachievements.org/gameList.php?c=4" + ;; + +gamegear) + readonly URL="https://retroachievements.org/gameList.php?c=15" + ;; + +mastersystem) + readonly URL="https://retroachievements.org/gameList.php?c=11" + ;; + +megadrive) + readonly URL="https://retroachievements.org/gameList.php?c=1" + ;; + +msx) + readonly URL="https://retroachievements.org/gameList.php?c=29" + ;; + +ngpc) + readonly URL="https://retroachievements.org/gameList.php?c=14" + ;; + +nes) + readonly URL="https://retroachievements.org/gameList.php?c=7" + ;; + +n64) + readonly URL="https://retroachievements.org/gameList.php?c=2" + ;; + +nds) + readonly URL="https://retroachievements.org/gameList.php?c=18" + ;; + +panasonic3do) + readonly URL="https://retroachievements.org/gameList.php?c=43" + ;; + +pc8000) + readonly URL="https://retroachievements.org/gameList.php?c=47" + ;; + +pcengine) + readonly URL="https://retroachievements.org/gameList.php?c=8" + ;; + +pcfx) + readonly URL="https://retroachievements.org/gameList.php?c=49" + ;; + +psx) + readonly URL="https://retroachievements.org/gameList.php?c=12" + ;; + +pokemonmini) + readonly URL="https://retroachievements.org/gameList.php?c=24" + ;; + +sega32x) + readonly URL="https://retroachievements.org/gameList.php?c=10" + ;; + +segacd) + readonly URL="https://retroachievements.org/gameList.php?c=9" + ;; + +segasaturn) + readonly URL="https://retroachievements.org/gameList.php?c=39" + ;; + +sg1000) + readonly URL="https://retroachievements.org/gameList.php?c=33" + ;; + +snes) + readonly URL="https://retroachievements.org/gameList.php?c=3" + ;; + +vectrex) + readonly URL="https://retroachievements.org/gameList.php?c=46" + ;; + +virtualboy) + readonly URL="https://retroachievements.org/gameList.php?c=28" + ;; + +wonderswan) + readonly URL="https://retroachievements.org/gameList.php?c=53" + ;; + +esac + +wget "$URL" -O "$SYSTEM_HTML" + +# process system html to get game url list +printf "%s\r" "${BLUE_COLOR}Processing $SYSTEM_NAME. ${RESET_COLOR}" +grep -E "Game/[0-9]+" -Eo "$SYSTEM_HTML" | sort -u | sed 's,Game/,https://retroachievements.org/linkedhashes.php?g=,' > "$URL_LIST" +echo "Found $(wc -l <"$URL_LIST") games." + +# create system folder for webpages with hashes +if test -d "./${GAME_LIST_DIR}/$SYSTEM_NAME"; then + rm -rdf "./${GAME_LIST_DIR}/$SYSTEM_NAME" + echo "Deleting ${GAME_LIST_DIR}/$SYSTEM_NAME folder." +fi +if ! test -d "./${GAME_LIST_DIR}/$SYSTEM_NAME"; then + mkdir -p "./${GAME_LIST_DIR}/$SYSTEM_NAME" + echo "Create ${GAME_LIST_DIR}/$SYSTEM_NAME folder." +fi + +# download webpages with hashes form url list +echo "Downloading game info ..." +wget --random-wait --no-verbose --load-cookies "${COOKIE_FILE}" -i "${URL_LIST}" -P "${GAME_LIST_DIR}/${SYSTEM_NAME}" + +# remove tempfiles +if test -f "$URL_LIST"; then + rm "$URL_LIST" +fi + +# extract hashes from files on data/system folder + +echo "Processing $SYSTEM_NAME system." +echo "Working folder: $GAME_LIST_DIR" + +# if file exists, delete it +if test -f "${HASCHEEVOS_TXT}"; then + rm "${HASCHEEVOS_TXT}" +fi + +# if file exists, delete it +if test -f "${HASHLIBRARY_JSON}"; then + rm "${HASHLIBRARY_JSON}" +fi + +# check if folder exists +if ! test -d "${GAME_LIST_DIR}/${SYSTEM_NAME}"; then + echo "Folder ${GAME_LIST_DIR}/${SYSTEM_NAME} does not exist." + exit 0 +fi + +# create HASCHEEVOS_TXT file +grep -E "title=\"(.*?)\"" -Eo "${GAME_LIST_DIR}/${SYSTEM_NAME}/"* >> "${HASCHEEVOS_TXT}" +sed -i -E 's,^\/(.*?)g=,,g' "${HASCHEEVOS_TXT}" +sed -i -E 's,title=,true:,g' "${HASCHEEVOS_TXT}" +echo "File $HASCHEEVOS_TXT created." + +# create HASHLIBRARY_JSON +echo "{" > "${HASHLIBRARY_JSON}" +echo " \"Success\": true," >> "${HASHLIBRARY_JSON}" +echo " \"MD5List\": {" >> "${HASHLIBRARY_JSON}" +for i in "${GAME_LIST_DIR}/${SYSTEM_NAME}/"*; do + # catch hash and append to file + printf "%s\n" "$(grep -E "[A-Fa-f0-9]{32}" -Ho "$i")" >> "${HASHLIBRARY_JSON}" + printf "%s\r" "${BLUE_COLOR}Parsing $(basename "$i") ... ${RESET_COLOR}" +done +perl -i -pe 's/(^\/.*?=)([0-9]+):([0-9A-Za-z]{32})/ "$3": $2,/' "${HASHLIBRARY_JSON}" +echo " }" >> "${HASHLIBRARY_JSON}" +echo "}" >> "${HASHLIBRARY_JSON}" +perl -i -00pe 's/,(?!.*,)//s' "${HASHLIBRARY_JSON}" +printf "%s\n" "Parsed $(wc -l < "${HASHLIBRARY_JSON}") hashes from $(find "${GAME_LIST_DIR}"/"$1" -type f | wc -l) games. " +echo "File: $HASHLIBRARY_JSON created." diff --git a/update_cookie.sh b/update_cookie.sh new file mode 100755 index 0000000..2e84660 --- /dev/null +++ b/update_cookie.sh @@ -0,0 +1,2 @@ +#!/bin/bash +lynx https://www.retroachievements.org diff --git a/webdata/atari7800.html b/webdata/atari7800.html new file mode 100644 index 0000000..d9f20f3 --- /dev/null +++ b/webdata/atari7800.html @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + +Supported Games Atari 7800 - RetroAchievements +
+
Forgot password?
login to RetroAchievements:
User: 
Pass: 
...or create a new account

+
+
+ All Games » Atari 7800
+
+

Atari 7800

TitleAchievementsPointsLeaderboards
Donkey Kong
14143 (753)3
Joust
57530 (4535)4
Ms. Pac-Man
18175 (846)1
Ninja Golf
31485 (3243)
Rampage
15320 (4767)1
Robotron 2084
43535 (8987)
Totals: 6 games1782188 (23131)9

+
+
+
+
+ \ No newline at end of file diff --git a/webdata/atari7800/linkedhashes.php?g=13352 b/webdata/atari7800/linkedhashes.php?g=13352 new file mode 100644 index 0000000..7756029 --- /dev/null +++ b/webdata/atari7800/linkedhashes.php?g=13352 @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + +Linked Hashes - RetroAchievements +
+

JailDesignerJailDesigner (29088) (67289)
logout
 (0)


+
+ +

List of Linked Hashes

+ +
Ninja Golf (Atari 7800)


Hashes are used to confirm if two copies of a file are identical. We use it to ensure the player is using the same ROM as the achievement developer, or a compatible one.

Currently this game has 2 unique ROM(s) registered for it with the following MD5s:



+
+
+
+ \ No newline at end of file diff --git a/webdata/atari7800/linkedhashes.php?g=13354 b/webdata/atari7800/linkedhashes.php?g=13354 new file mode 100644 index 0000000..635122a --- /dev/null +++ b/webdata/atari7800/linkedhashes.php?g=13354 @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + +Linked Hashes - RetroAchievements +
+

JailDesignerJailDesigner (29088) (67289)
logout
 (0)


+
+ +

List of Linked Hashes

+ +
Donkey Kong (Atari 7800)


Hashes are used to confirm if two copies of a file are identical. We use it to ensure the player is using the same ROM as the achievement developer, or a compatible one.

Currently this game has 2 unique ROM(s) registered for it with the following MD5s:



+
+
+
+ \ No newline at end of file diff --git a/webdata/atari7800/linkedhashes.php?g=13364 b/webdata/atari7800/linkedhashes.php?g=13364 new file mode 100644 index 0000000..a4685e2 --- /dev/null +++ b/webdata/atari7800/linkedhashes.php?g=13364 @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + +Linked Hashes - RetroAchievements +
+

JailDesignerJailDesigner (29088) (67289)
logout
 (0)


+
+ +

List of Linked Hashes

+ +
Joust (Atari 7800)


Hashes are used to confirm if two copies of a file are identical. We use it to ensure the player is using the same ROM as the achievement developer, or a compatible one.

Currently this game has 1 unique ROM(s) registered for it with the following MD5s:



+
+
+
+ \ No newline at end of file diff --git a/webdata/atari7800/linkedhashes.php?g=13368 b/webdata/atari7800/linkedhashes.php?g=13368 new file mode 100644 index 0000000..e6a2cbe --- /dev/null +++ b/webdata/atari7800/linkedhashes.php?g=13368 @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + +Linked Hashes - RetroAchievements +
+

JailDesignerJailDesigner (29088) (67289)
logout
 (0)


+
+ +

List of Linked Hashes

+ +
Ms. Pac-Man (Atari 7800)


Hashes are used to confirm if two copies of a file are identical. We use it to ensure the player is using the same ROM as the achievement developer, or a compatible one.

Currently this game has 2 unique ROM(s) registered for it with the following MD5s:



+
+
+
+ \ No newline at end of file diff --git a/webdata/atari7800/linkedhashes.php?g=13371 b/webdata/atari7800/linkedhashes.php?g=13371 new file mode 100644 index 0000000..d2c6cc2 --- /dev/null +++ b/webdata/atari7800/linkedhashes.php?g=13371 @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + +Linked Hashes - RetroAchievements +
+

JailDesignerJailDesigner (29088) (67289)
logout
 (0)


+
+ +

List of Linked Hashes

+ +
Rampage (Atari 7800)


Hashes are used to confirm if two copies of a file are identical. We use it to ensure the player is using the same ROM as the achievement developer, or a compatible one.

Currently this game has 1 unique ROM(s) registered for it with the following MD5s:



+
+
+
+ \ No newline at end of file diff --git a/webdata/atari7800/linkedhashes.php?g=13372 b/webdata/atari7800/linkedhashes.php?g=13372 new file mode 100644 index 0000000..f4999cc --- /dev/null +++ b/webdata/atari7800/linkedhashes.php?g=13372 @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + +Linked Hashes - RetroAchievements +
+

JailDesignerJailDesigner (29088) (67289)
logout
 (0)


+
+ +

List of Linked Hashes

+ +
Robotron 2084 (Atari 7800)


Hashes are used to confirm if two copies of a file are identical. We use it to ensure the player is using the same ROM as the achievement developer, or a compatible one.

Currently this game has 1 unique ROM(s) registered for it with the following MD5s:



+
+
+
+ \ No newline at end of file