From 0a0c5c11a857772aada9b9fe95fb1f03342a09e9 Mon Sep 17 00:00:00 2001 From: Sergio Date: Mon, 4 Sep 2023 22:04:49 +0200 Subject: [PATCH] zxdb.py: reescrito el bucle principal --- zxdb.py | 147 +++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 113 insertions(+), 34 deletions(-) diff --git a/zxdb.py b/zxdb.py index 4c4238f..87d4a8d 100644 --- a/zxdb.py +++ b/zxdb.py @@ -14,7 +14,7 @@ from urllib.request import urlretrieve url_prefix = [ r"https://spectrumcomputing.co.uk", - r"https://archive.org/download/World_of_Spectrum_June_2017_Mirror/World%20of%20Spectrum%20June%202017%20Mirror.zip/World%20of%20Spectrum%20June%202017%20Mirrorb", + r"https://archive.org/download/World_of_Spectrum_June_2017_Mirror/World%20of%20Spectrum%20June%202017%20Mirror.zip/World%20of%20Spectrum%20June%202017%20Mirror", ] destination_path = r"/home/sergio/zx/zxdb/games/" cache_path = r"/home/sergio/zx/zxdb/cache/games/" @@ -22,7 +22,7 @@ wait = False # Establece una pausa aleatoria entre descargas min_wait = 3 # Segundos mínimos a esperar entre descargas max_wait = 5 # Segundos máximos a esperar entre descargas elements = [] -filetypes = [ +filetypes_on_root = [ "Tape image", "Disk image", "Snapshot image", @@ -73,7 +73,14 @@ def select3(cursor): e.title;""" cursor.execute(query) for row in cursor: - elements.append(list(row)) + element = dict( + title=row[0], + developer=row[1], + release_year=row[2], + url=row[3], + filetype=row[4], + ) + elements.append(element) def connect(): @@ -104,6 +111,43 @@ def connect(): cursor.close() +def process_elements(): + global elements + for i in range(len(elements)): + # Construye el nombre de la carpeta raiz + elements[i]["root_folder"] = ( + elements[i]["title"] + + " (" + + str(elements[i]["release_year"]) + + ")(" + + elements[i]["developer"] + + ")" + ) + + # Obtiene el nombre del fichero a partir de la url de descarga + elements[i]["file_name"] = url_filename(elements[i]["url"]) + + # Establece la subcarpeta dentro de la raiz + elements[i]["subfolder"] = "" + if elements[i]["filetype"] not in filetypes_on_root: + elements[i]["subfolder"] = normalize_path(elements[i]["filetype"]) + + # Averigua si el fichero está en formato .zip + elements[i]["is_zip"] = elements[i]["file_name"].endswith(".zip") + + # Calcula el nombre del fichero si es un zip + elements[i]["non_zip_file_name"] = elements[i]["file_name"] + if elements[i]["is_zip"]: + elements[i]["non_zip_file_name"] = elements[i]["file_name"][:-4] + + # Añade el prefijo a la url + if elements[i]["url"].startswith("/zxdb"): + elements[i]["url"] = url_prefix[0] + str(elements[i]["url"]) + elif elements[i]["url"].startswith("/pub"): + #elements[i]["url"] = url_prefix[1] + str(elements[i]["url"][4:]) + elements[i]["url"] = url_prefix[0] + str(elements[i]["url"]) + + def add_prefix(): global elements for i in range(len(elements)): @@ -158,32 +202,34 @@ def get_files(): last_game_folder = "" for element in elements: # Fichero a descargar - downloaded_file = url_filename(element[3]) + # downloaded_file = url_filename(element[3]) # Carpeta del juego en destino y en caché - game_folder = element[0] + " (" + str(element[2]) + ")(" + element[1] + ")" - destination_folder = os.path.join(destination_path, game_folder) - if not os.path.isdir(destination_folder): - os.mkdir(destination_folder) - cache_folder = os.path.join(cache_path, game_folder) - if not os.path.isdir(cache_folder): - os.mkdir(cache_folder) + game_folder = element["root_folder"] + destination_folder = os.path.join(destination_path, element["root_folder"]) + destination_subfolder = os.path.join(destination_folder, element["subfolder"]) + # if not os.path.isdir(destination_folder): + # os.mkdir(destination_folder) + cache_folder = os.path.join(cache_path, element["root_folder"]) + cache_subfolder = os.path.join(cache_folder, element["subfolder"]) + # if not os.path.isdir(cache_folder): + # os.mkdir(cache_folder) # Carpeta de tipo de fichero en destino y en caché - if element[4] not in filetypes: - filetype_folder = normalize_path(element[4]) - destination_folder = os.path.join( - destination_path, game_folder, filetype_folder - ) - if not os.path.isdir(destination_folder): - os.mkdir(destination_folder) - cache_folder = os.path.join(cache_path, game_folder, filetype_folder) - if not os.path.isdir(cache_folder): - os.mkdir(cache_folder) + # if element[4] not in filetypes: + # filetype_folder = normalize_path(element[4]) + # destination_folder = os.path.join( + # destination_path, game_folder, filetype_folder + # ) + # if not os.path.isdir(destination_folder): + # os.mkdir(destination_folder) + # cache_folder = os.path.join(cache_path, game_folder, filetype_folder) + # if not os.path.isdir(cache_folder): + # os.mkdir(cache_folder) # Ruta completa hasta el fichero de destino y de caché - destination_file = os.path.join(destination_folder, downloaded_file) - cache_file = os.path.join(cache_folder, downloaded_file) + destination_file = os.path.join(destination_subfolder, element["file_name"]) + cache_file = os.path.join(cache_subfolder, element["file_name"]) # Actualiza las variables de presentación current_file = current_file + 1 @@ -196,37 +242,69 @@ def get_files(): if not os.path.isfile(destination_file): # Comprueba si ya existe el fichero en la cache if os.path.isfile(cache_file): + # Si encuentra el fichero en cache, crea las carpetas de destino y lo copia o lo extrae + if not os.path.isdir(destination_folder): + os.mkdir(destination_folder) + if not os.path.isdir(destination_subfolder): + os.mkdir(destination_subfolder) if cache_file.endswith(".zip"): - unzip_file(cache_file, destination_folder) + unzip_file(cache_file, destination_subfolder) else: shutil.copyfile(cache_file, destination_file) print( "({:{width}} / {}) : cached : {} ({})".format( - current_file, total_files, downloaded_file, element[4], width=2 + current_file, + total_files, + element["file_name"], + element["filetype"], + width=2, ) ) # El fichero no está en la cache else: status = "not found " - if download_file(element[3], cache_file): + if download_file(element["url"], "downloaded_file.tmp"): status = "downloaded" + if os.path.isfile("downloaded_file.tmp"): + # Copia el fichero temnporal a la cache + if not os.path.isdir(cache_folder): + os.mkdir(cache_folder) + if not os.path.isdir(cache_subfolder): + os.mkdir(cache_subfolder) + shutil.copyfile("downloaded_file.tmp", cache_file) + os.remove('downloaded_file.tmp') + # Copia el fichero de la cache al destino + if os.path.isfile(cache_file): + if not os.path.isdir(destination_folder): + os.mkdir(destination_folder) + if not os.path.isdir(destination_subfolder): + os.mkdir(destination_subfolder) + if cache_file.endswith(".zip"): + unzip_file(cache_file, destination_folder) + else: + shutil.copyfile(cache_file, destination_file) print( "({:{width}} / {}) : {} : {} ({})".format( - current_file, total_files, status, downloaded_file, element[4], width=2 + current_file, + total_files, + status, + element["file_name"], + element["filetype"], + width=2, ) ) - if os.path.isfile(cache_file): - if cache_file.endswith(".zip"): - unzip_file(cache_file, destination_folder) - else: - shutil.copyfile(cache_file, destination_file) if wait: time.sleep(random.randint(min_wait, max_wait)) + # El fichero ya existe en el destino else: print( "({:{width}} / {}) : skipping : {} ({})".format( - current_file, total_files, downloaded_file, element[4], width=2 + current_file, + total_files, + element["file_name"], + element["filetype"], + width=2, ) ) @@ -241,7 +319,8 @@ def normalize_path(path): def main(): connect() - add_prefix() + # add_prefix() + process_elements() #for element in elements: # print(element)