Afegit retry per a les descàrregues
This commit is contained in:
@@ -14,6 +14,8 @@ from mysql.connector import errorcode
|
||||
from unidecode import unidecode
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import urlretrieve
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
# Cargar las variables de entorno desde el archivo .env
|
||||
load_dotenv()
|
||||
@@ -212,9 +214,7 @@ def process_elements():
|
||||
|
||||
# Añade el prefijo a la url
|
||||
if elements[i]["url"].startswith("/zxdb"):
|
||||
elements[i]["url"] = url_prefix["spectrum_computing"] + str(
|
||||
elements[i]["url"]
|
||||
)
|
||||
elements[i]["url"] = url_prefix["spectrum_computing"] + str(elements[i]["url"])
|
||||
elif elements[i]["url"].startswith("/pub"):
|
||||
elements[i]["url"] = url_prefix["wos"] + str(elements[i]["url"][4:])
|
||||
elif elements[i]["url"].startswith("/nvg"):
|
||||
@@ -228,7 +228,7 @@ def url_filename(url):
|
||||
return filename
|
||||
|
||||
# Descarga un fichero a partir de una URL
|
||||
def download_file(url, dest):
|
||||
'''def download_file(url, dest):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
if r.status_code != 200:
|
||||
@@ -247,7 +247,24 @@ def download_file(url, dest):
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
# catastrophic error. bail.
|
||||
raise SystemExit(e)
|
||||
raise SystemExit(e)'''
|
||||
|
||||
|
||||
|
||||
def download_file(url, destination):
|
||||
session = requests.Session()
|
||||
retries = Retry(total=3, backoff_factor=1, status_forcelist=[500, 502, 503, 504])
|
||||
session.mount('https://', HTTPAdapter(max_retries=retries))
|
||||
|
||||
try:
|
||||
response = session.get(url, timeout=10)
|
||||
response.raise_for_status()
|
||||
with open(destination, 'wb') as file:
|
||||
file.write(response.content)
|
||||
return True
|
||||
except requests.exceptions.RequestException as e:
|
||||
logging.error(f"Error al descargar el archivo: {e}")
|
||||
return False
|
||||
|
||||
# Descomprime los ficheros que coinciden con la lista de extensiones
|
||||
def unzip_file(src, dst):
|
||||
|
||||
Reference in New Issue
Block a user