Afegit retry per a les descàrregues

This commit is contained in:
2024-11-13 13:53:30 +01:00
parent 84c1495bc6
commit ceaa55b44e
+22 -5
View File
@@ -14,6 +14,8 @@ from mysql.connector import errorcode
from unidecode import unidecode
from urllib.parse import urlparse
from urllib.request import urlretrieve
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
# Cargar las variables de entorno desde el archivo .env
load_dotenv()
@@ -212,9 +214,7 @@ def process_elements():
# Añade el prefijo a la url
if elements[i]["url"].startswith("/zxdb"):
elements[i]["url"] = url_prefix["spectrum_computing"] + str(
elements[i]["url"]
)
elements[i]["url"] = url_prefix["spectrum_computing"] + str(elements[i]["url"])
elif elements[i]["url"].startswith("/pub"):
elements[i]["url"] = url_prefix["wos"] + str(elements[i]["url"][4:])
elif elements[i]["url"].startswith("/nvg"):
@@ -228,7 +228,7 @@ def url_filename(url):
return filename
# Descarga un fichero a partir de una URL
def download_file(url, dest):
'''def download_file(url, dest):
try:
r = requests.get(url)
if r.status_code != 200:
@@ -247,7 +247,24 @@ def download_file(url, dest):
except requests.exceptions.RequestException as e:
# catastrophic error. bail.
raise SystemExit(e)
raise SystemExit(e)'''
def download_file(url, destination):
session = requests.Session()
retries = Retry(total=3, backoff_factor=1, status_forcelist=[500, 502, 503, 504])
session.mount('https://', HTTPAdapter(max_retries=retries))
try:
response = session.get(url, timeout=10)
response.raise_for_status()
with open(destination, 'wb') as file:
file.write(response.content)
return True
except requests.exceptions.RequestException as e:
logging.error(f"Error al descargar el archivo: {e}")
return False
# Descomprime los ficheros que coinciden con la lista de extensiones
def unzip_file(src, dst):