o
    Ďi9                     @   s   d dl Zd dlZd dlZd dlmZ d dlZd dlZd dl	m
Z
 d dlmZ d dlmZ edZejdZeeZdd	 Zd
d Zdd Zdd ZdS )    N)Path)NoCredentialsError)tqdm)colorstrAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYc                 C   sR   |  dr| dd  } | d}|d }d}t|dkr%d|dd  }||fS )Nzs3://   /r       )
startswithsplitlenjoin)s3_uris3_componentsbuckets3_key r   L/home/jeff/fluffinator/venv/lib/python3.10/site-packages/yolov5/utils/aws.pyparse_s3_uri   s   

r   c                 C   s   t jdttd}t|\}}z
|| || W dS  ty-   ttd d|   Y dS  t	y?   ttd d Y dS w )Ns3aws_access_key_idaws_secret_access_keyTaws:z0 S3 upload failed because local file not found: FzY AWS credentials are not set. Please configure aws via CLI or set required ENV variables.)
boto3clientr   r   r   upload_fileFileNotFoundErrorprintr   r   )
local_files3_filer   r   r   r   r   r   upload_file_to_s3   s   r#   c                 C   s0   z
| j ||d W dS    | ||| Y dS )N)BucketKeyr   r   )head_objectr   )r   r   
local_paths3_pathr   r   r   upload_single_file/   s   r)   c              
   C   s8  t jdttd}t|\}}g }g }g }g }t| D ]5\}	}
}|D ]-}tj|	|}|}tj	|| }tj||}|
| |
| |
| |
| q"qtjjdd}tt|t||||t|td dd}W d    n1 szw   Y  tt|}|dkrttd d	| d
 d S d S )Nr   r   
   )max_workersr   z Uploading dataset to S3)totaldescr   z	 Skipped z! items since they already exists.)r   r   r   r   r   oswalkpathr   relpathappend
concurrentfuturesThreadPoolExecutorlistr   mapr)   r   r   npsumarrayLOGGERwarning)local_folder	s3_folderr   r   r   client_listbucket_listlocal_path_lists3_path_listrootdirsfilesfilenamer'   r(   relative_path	conc_execalready_exist_listnum_already_existr   r   r   upload_folder_to_s38   s<   



rK   )concurrent.futuresr3   loggingr.   pathlibr   r   numpyr8   botocore.exceptionsr   r   yolov5.utils.generalr   getenvr   environgetr   	getLogger__name__r;   r   r#   r)   rK   r   r   r   r   <module>   s     

	