diff --git a/zxdb.py b/zxdb.py index 87d4a8d..d40d72e 100644 --- a/zxdb.py +++ b/zxdb.py @@ -12,15 +12,17 @@ from urllib.parse import urlparse from urllib.request import urlretrieve -url_prefix = [ - r"https://spectrumcomputing.co.uk", - r"https://archive.org/download/World_of_Spectrum_June_2017_Mirror/World%20of%20Spectrum%20June%202017%20Mirror.zip/World%20of%20Spectrum%20June%202017%20Mirror", -] +url_prefix = { + "spectrum_computing": r"https://spectrumcomputing.co.uk", + "wos": r"https://php.sustancia.synology.me/wos", + "nvg": r"https://php.sustancia.synology.me/nvg", +} destination_path = r"/home/sergio/zx/zxdb/games/" cache_path = r"/home/sergio/zx/zxdb/cache/games/" +temp_file = r"/tmp/zxdb.download.tmp" wait = False # Establece una pausa aleatoria entre descargas -min_wait = 3 # Segundos mínimos a esperar entre descargas -max_wait = 5 # Segundos máximos a esperar entre descargas +min_wait = 1 # Segundos mínimos a esperar entre descargas +max_wait = min_wait + 1 # Segundos máximos a esperar entre descargas elements = [] filetypes_on_root = [ "Tape image", @@ -66,11 +68,13 @@ def select3(cursor): e.id = r.entry_id AND p.release_seq = r.release_seq) WHERE - e.title = 'Afterburner' AND + (e.availabletype_id = 'A' OR e.availabletype_id = 'D') AND + f.text <> 'Remote link' AND r.release_seq = 0 AND - g.text like '%Game%' + (g.text like '%Game:%' AND g.text not like 'Casual%') ORDER BY e.title;""" + #(l.country_id = 'ES' AND l.labeltype_id = 'Z') AND cursor.execute(query) for row in cursor: element = dict( @@ -123,38 +127,32 @@ def process_elements(): + elements[i]["developer"] + ")" ) - + # Obtiene el nombre del fichero a partir de la url de descarga elements[i]["file_name"] = url_filename(elements[i]["url"]) - + # Establece la subcarpeta dentro de la raiz elements[i]["subfolder"] = "" if elements[i]["filetype"] not in filetypes_on_root: elements[i]["subfolder"] = normalize_path(elements[i]["filetype"]) - + # Averigua si el fichero está en formato .zip elements[i]["is_zip"] = elements[i]["file_name"].endswith(".zip") - + # Calcula el nombre del fichero si es un zip elements[i]["non_zip_file_name"] = elements[i]["file_name"] if elements[i]["is_zip"]: elements[i]["non_zip_file_name"] = elements[i]["file_name"][:-4] - + # Añade el prefijo a la url if elements[i]["url"].startswith("/zxdb"): - elements[i]["url"] = url_prefix[0] + str(elements[i]["url"]) + elements[i]["url"] = url_prefix["spectrum_computing"] + str( + elements[i]["url"] + ) elif elements[i]["url"].startswith("/pub"): - #elements[i]["url"] = url_prefix[1] + str(elements[i]["url"][4:]) - elements[i]["url"] = url_prefix[0] + str(elements[i]["url"]) - - -def add_prefix(): - global elements - for i in range(len(elements)): - if elements[i][3].startswith("/zxdb"): - elements[i][3] = url_prefix[0] + str(elements[i][3]) - elif elements[i][3].startswith("/pub"): - elements[i][3] = url_prefix[1] + str(elements[i][3][4:]) + elements[i]["url"] = url_prefix["wos"] + str(elements[i]["url"][4:]) + elif elements[i]["url"].startswith("/nvg"): + elements[i]["url"] = url_prefix["nvg"] + str(elements[i]["url"][4:]) def url_filename(url): @@ -166,11 +164,7 @@ def url_filename(url): def download_file(url, dest): try: - print("") - print("URL : {}".format(url)) r = requests.get(url) - print("STATUS_CODE: {}".format(r.status_code)) - print("RESPONSE : {}".format(r.reason)) if r.status_code != 200: return False with open(dest, "wb") as f: @@ -191,41 +185,33 @@ def download_file(url, dest): def unzip_file(src, dst): - with zipfile.ZipFile(src, "r") as zip_ref: - zip_ref.extractall(dst) + # with zipfile.ZipFile(src, "r") as zip_ref: + # zip_ref.extractall(dst) + archive = src + directory = dst + extensions = (".z80", ".sna", ".tzx", ".tap", "dsk", ".trd", ".Z80", ".SNA", ".TZX", ".TAP", "DSK", ".TRD") + zip_file = zipfile.ZipFile(archive, "r") + [ + zip_file.extract(file, directory) + for file in zip_file.namelist() + if file.endswith(extensions) + ] + zip_file.close() def get_files(): # Variables para la presentación en pantalla de la descarga current_file = 0 total_files = len(elements) + total_files_width = len(str(total_files)) last_game_folder = "" for element in elements: - # Fichero a descargar - # downloaded_file = url_filename(element[3]) - # Carpeta del juego en destino y en caché game_folder = element["root_folder"] destination_folder = os.path.join(destination_path, element["root_folder"]) destination_subfolder = os.path.join(destination_folder, element["subfolder"]) - # if not os.path.isdir(destination_folder): - # os.mkdir(destination_folder) cache_folder = os.path.join(cache_path, element["root_folder"]) cache_subfolder = os.path.join(cache_folder, element["subfolder"]) - # if not os.path.isdir(cache_folder): - # os.mkdir(cache_folder) - - # Carpeta de tipo de fichero en destino y en caché - # if element[4] not in filetypes: - # filetype_folder = normalize_path(element[4]) - # destination_folder = os.path.join( - # destination_path, game_folder, filetype_folder - # ) - # if not os.path.isdir(destination_folder): - # os.mkdir(destination_folder) - # cache_folder = os.path.join(cache_path, game_folder, filetype_folder) - # if not os.path.isdir(cache_folder): - # os.mkdir(cache_folder) # Ruta completa hasta el fichero de destino y de caché destination_file = os.path.join(destination_subfolder, element["file_name"]) @@ -239,7 +225,11 @@ def get_files(): last_game_folder = game_folder # Comprueba si ya existe el fichero a descargar - if not os.path.isfile(destination_file): + if not os.path.isfile(destination_file) and ( + not os.path.isfile( + os.path.join(destination_subfolder, element["non_zip_file_name"]) + ) + ): # Comprueba si ya existe el fichero en la cache if os.path.isfile(cache_file): # Si encuentra el fichero en cache, crea las carpetas de destino y lo copia o lo extrae @@ -247,7 +237,7 @@ def get_files(): os.mkdir(destination_folder) if not os.path.isdir(destination_subfolder): os.mkdir(destination_subfolder) - if cache_file.endswith(".zip"): + if cache_file.endswith(".zip") and element["subfolder"] == "": unzip_file(cache_file, destination_subfolder) else: shutil.copyfile(cache_file, destination_file) @@ -257,29 +247,32 @@ def get_files(): total_files, element["file_name"], element["filetype"], - width=2, + width=total_files_width, ) ) # El fichero no está en la cache else: status = "not found " - if download_file(element["url"], "downloaded_file.tmp"): + if download_file(element["url"], temp_file): status = "downloaded" - if os.path.isfile("downloaded_file.tmp"): + if os.path.isfile(temp_file): # Copia el fichero temnporal a la cache if not os.path.isdir(cache_folder): os.mkdir(cache_folder) if not os.path.isdir(cache_subfolder): os.mkdir(cache_subfolder) - shutil.copyfile("downloaded_file.tmp", cache_file) - os.remove('downloaded_file.tmp') + shutil.copyfile(temp_file, cache_file) + os.remove(temp_file) # Copia el fichero de la cache al destino if os.path.isfile(cache_file): if not os.path.isdir(destination_folder): os.mkdir(destination_folder) if not os.path.isdir(destination_subfolder): os.mkdir(destination_subfolder) - if cache_file.endswith(".zip"): + if ( + cache_file.endswith(".zip") + and element["subfolder"] == "" + ): unzip_file(cache_file, destination_folder) else: shutil.copyfile(cache_file, destination_file) @@ -290,7 +283,7 @@ def get_files(): status, element["file_name"], element["filetype"], - width=2, + width=total_files_width, ) ) if wait: @@ -304,7 +297,7 @@ def get_files(): total_files, element["file_name"], element["filetype"], - width=2, + width=total_files_width, ) ) @@ -319,13 +312,17 @@ def normalize_path(path): def main(): connect() - # add_prefix() process_elements() - #for element in elements: - # print(element) + # for element in elements: + # for key, value in element.items(): + # print(key, ':', value) get_files() + # for element in elements: + # print(element['title']) + + # print(len(elements)) if __name__ == "__main__":