Afegit retry per a les descàrregues
This commit is contained in:
@@ -14,6 +14,8 @@ from mysql.connector import errorcode
|
|||||||
from unidecode import unidecode
|
from unidecode import unidecode
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
from urllib.request import urlretrieve
|
from urllib.request import urlretrieve
|
||||||
|
from requests.adapters import HTTPAdapter
|
||||||
|
from urllib3.util.retry import Retry
|
||||||
|
|
||||||
# Cargar las variables de entorno desde el archivo .env
|
# Cargar las variables de entorno desde el archivo .env
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
@@ -212,9 +214,7 @@ def process_elements():
|
|||||||
|
|
||||||
# Añade el prefijo a la url
|
# Añade el prefijo a la url
|
||||||
if elements[i]["url"].startswith("/zxdb"):
|
if elements[i]["url"].startswith("/zxdb"):
|
||||||
elements[i]["url"] = url_prefix["spectrum_computing"] + str(
|
elements[i]["url"] = url_prefix["spectrum_computing"] + str(elements[i]["url"])
|
||||||
elements[i]["url"]
|
|
||||||
)
|
|
||||||
elif elements[i]["url"].startswith("/pub"):
|
elif elements[i]["url"].startswith("/pub"):
|
||||||
elements[i]["url"] = url_prefix["wos"] + str(elements[i]["url"][4:])
|
elements[i]["url"] = url_prefix["wos"] + str(elements[i]["url"][4:])
|
||||||
elif elements[i]["url"].startswith("/nvg"):
|
elif elements[i]["url"].startswith("/nvg"):
|
||||||
@@ -228,7 +228,7 @@ def url_filename(url):
|
|||||||
return filename
|
return filename
|
||||||
|
|
||||||
# Descarga un fichero a partir de una URL
|
# Descarga un fichero a partir de una URL
|
||||||
def download_file(url, dest):
|
'''def download_file(url, dest):
|
||||||
try:
|
try:
|
||||||
r = requests.get(url)
|
r = requests.get(url)
|
||||||
if r.status_code != 200:
|
if r.status_code != 200:
|
||||||
@@ -247,7 +247,24 @@ def download_file(url, dest):
|
|||||||
|
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
# catastrophic error. bail.
|
# catastrophic error. bail.
|
||||||
raise SystemExit(e)
|
raise SystemExit(e)'''
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def download_file(url, destination):
|
||||||
|
session = requests.Session()
|
||||||
|
retries = Retry(total=3, backoff_factor=1, status_forcelist=[500, 502, 503, 504])
|
||||||
|
session.mount('https://', HTTPAdapter(max_retries=retries))
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = session.get(url, timeout=10)
|
||||||
|
response.raise_for_status()
|
||||||
|
with open(destination, 'wb') as file:
|
||||||
|
file.write(response.content)
|
||||||
|
return True
|
||||||
|
except requests.exceptions.RequestException as e:
|
||||||
|
logging.error(f"Error al descargar el archivo: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
# Descomprime los ficheros que coinciden con la lista de extensiones
|
# Descomprime los ficheros que coinciden con la lista de extensiones
|
||||||
def unzip_file(src, dst):
|
def unzip_file(src, dst):
|
||||||
|
|||||||
Reference in New Issue
Block a user