HEX

File: //lib/python3.6/site-packages/S3/__pycache__/FileLists.cpython-36.pyc
3

�9pe�p�@s:ddlmZddlmZddlmZddlmZddlmZddlmZmZm	Z	ddl
mZmZm
Z
mZdd	lmZdd
lmZddlmZmZmZddlZddlZddlZddlZddlZddlZdd
lmZejd"kZdddgZ dd�Z!dd�Z"dd�Z#dd�Z$dd�Z%dd�Z&d#dd�Z'ddifd d�Z(d!d�Z)dS)$�)�absolute_import�)�S3)�Config)�S3Uri)�FileDict)�dateS3toUnix�dateRFC822toUnix�s3path)�	unicodise�deunicodise�
deunicodise_s�replace_nonprintables)�ParameterError)�	HashCache)�debug�info�warningN)�S_ISDIR��fetch_local_list�fetch_remote_list�compare_filelistsccs�ytjt|��}Wntk
r&dSXgg}}xN|D]F}t|�}tjjttjj||���rtt||�s~|j	|�q8|j	|�q8W|||fVxB|D]:}tjj||�}tjj
t|��s�xt|�D]
}|Vq�Wq�WdS)z_
    Reimplementation of python's os.walk to nicely support unicode in input as in output.
    N)�os�listdirr�	Exceptionr�path�isdir�join�handle_exclude_include_walk_dir�append�islink�_os_walk_unicode)�top�names�dirs�nondirs�name�new_path�x�r*�/usr/lib/python3.6/FileLists.pyr"'s"



r"cs�t�}d}tjj||d�}td|�d}xH|jD]>�t�fdd	�|D��sNq2�j|�r2d
}td|j��Pq2W|r�xZ|j	D]P�t�fdd	�|D��s�q�td
|�j
f��j|�r�d}td|j��Pq�W|r�td|�ntd|�|S)a

    Should this root/dirname directory be excluded? (otherwise included by default)
    Exclude dir matches in the current directory
    This prevents us from recursing down trees we know we want to ignore
    return True for excluding, and False for including
    �/)$�/)\Z�\/$�	\/\Z(?ms)�zCHECK: '%s'Fc3s|]}�jj|�VqdS)N)�pattern�endswith)�.0�dp)�rr*r+�	<genexpr>Osz2handle_exclude_include_walk_dir.<locals>.<genexpr>TzEXCL-MATCH: '%s'c3s|]}�jj|�VqdS)N)r1r2)r3r4)r5r*r+r6YszINCL-TEST: '%s' ~ %szINCL-MATCH: '%s'z
EXCLUDE: '%s'z
PASS: '%s')r,r-r.r/)rrrrr�exclude�any�search�
debug_exclude�includer1�
debug_include)�root�dirname�cfgZdirectory_patterns�d�excludedr*)r5r+r@s2

rc
cs�tjjt|��st�t|g�}x�t|�D]�\}}}ttjjt|���}xp|D]h}tjj	||�}ttjjt|���}tjj
t|��rL||ks�|j|tjj�r�t
d|�qL|j|�qLWq(Wx0|D](}	x"t|	�D]\}}}|||fVq�Wq�WdS)z�
    Walk filesystem, following symbolic links (but without recursion), on python2.4 and later

    If a symlink directory loop is detected, emit a warning and skip.
    E.g.: dir1/dir2/sym-dir -> ../dir2
    z+Skipping recursively symlinked directory %sN)rrrr�AssertionError�setr"r�realpathrr!�
startswith�sepr�add)
rZwalkdirs�dirpath�dirnames�	filenamesZreal_dirpathr>ZcurrentZreal_currentZwalkdirr*r*r+�_fswalk_follow_symlinkshs


rKccs(x"t|�D]\}}}|||fVq
WdS)z\
    Directory tree generator

    path (str) is the root of the directory tree to walk
    N)r")rrHrIrJr*r*r+�_fswalk_no_symlinks�srLcCs�td�t�}tdd�}x�|j�D]�}td|�d}x0|jD]&}|j|�r>d}td|j|�Pq>W|r�x0|jD]&}|j|�rtd}td|j|�PqtW|r�td|�||||<||=q"q"td	|�q"W||fS)
NzApplying --exclude/--includeF)�ignore_casezCHECK: '%s'TzEXCL-MATCH: '%s'zINCL-MATCH: '%s'z
EXCLUDE: '%s'z
PASS: '%s')	rrr�keysr7r9r:r;r<)�src_listr?�exclude_list�filerAr5r*r*r+�filter_exclude_include�s0


rRc
CsDdd�}i}x�|jD]�}z�d}|dkr>tjtjj�ddd�}nNytjt|�dd�}Wn6tk
r�}ztd||j	f�wWYdd}~XnXxb|D]Z}t
|�j�}tj
jtj
j||��}t
tj
jt|���}t
tj
jt|���}	||||	�q�WWd|�r|j�XqWg}
x2t|�D]&}||}|j�|
j|g|f��qW|
S)	NcSs&||kr|g||<n||j|�dS)N)r )r@�key�valuer*r*r+�_append�sz(_get_filelist_from_file.<locals>._append�-r5F)�mode�closefd)rWzJ--files-from input file %s could not be opened for reading (%s), skipping.)�
files_from�io�open�sys�stdin�filenor�IOErrorr�strerrorr�striprr�normpathrr>�basename�close�sorted�sortr )
r?�
local_pathrU�filelistZfname�f�e�liner>rc�resultrS�valuesr*r*r+�_get_filelist_from_file�s4
rnFcs^��fdd�}��fdd�}�fdd�}t��t���jrptjjt�j��rptjjt�j��dkrp�j�j�nt	d�g}t
d	d
�}d	}	t|�tt
tfkr�|g}|dkr��j}xN|D]F}
t|
�}|jdks�td|
��|j�r�|r�td
|
��|j|�q�Wx |D]}|||�|�\}}	�qWt|�dk�r4d	}	t|�\}}
||�}|�|�||	|
|fS)Ncs�t|�}d}td|�d}�xr|D�]h}|d7}|ddkrNtd||f�|dkrXq$||}|d}|d}ytjtjt|���}Wn4tk
r�}	z|	jtjkr�w$n�WYdd}	~	XnX|r�d}
n|j	}
||j
|
|j|j|j
|j|j|d	��||j	7}|�rq$d
�jkr$�j|j|j
|j|j	�}|dk�rvy|j|�}Wntk
�rZw$YnX�j|j|j
|j|j	|�|j||j|j
||j	�q$W|S)NrzYRunning stat() and reading/calculating MD5 values on %d files, this may take some time...ri�z[%d/%d]rV�	full_name�is_dir)�size�mtime�dev�inode�uid�gid�sr�md5)�lenrr�stat_result�statr�OSError�errno�ENOENT�st_size�update�st_mtime�st_dev�st_ino�st_uid�st_gid�sync_checksrx�get_md5r_rGZrecord_hardlink)�loc_listZlen_loc_list�
total_sizeZcounter�
relative_fileZ
loc_list_itemrorprwrjrqrx)�cacher?r*r+�_fetch_local_list_info�sV



z0fetch_local_list.<locals>._fetch_local_list_infocs\td�|j�dkrhytj�}tj�}Wntk
rDd}d}YnXddd||dddd�|d<|dfS|j�r�|j�}|j�}�r�t�j	�r�t
�|�}d}	qވjr�t|�}nt
|�}d}	n"d}|j�}|g|j�gfg}d}	�xr|D�]h\}
}}|
j||d�}
|�s|dff}n|df|dff}�x*|D�] \}}�x|D�]}tjj|
|�}|�r�tjjt|���r�tjjt|���r<td	|��q<tjjt|���r��j�s�td
|��q<tjj|
|�}tjjdk�r�dj|jtjj��}�jdk�r�t|�}|jd
��r|dd�}|�r6|�r6|ddk�r6|d7}||d�||<�q<W�q*Wq�W||	fS)Nz Compiling list of local files...rVrrF)rorqrrrurvrsrtrpTr0z"Skipping over non regular file: %szSkipping over symbolic link: %s�/Znormalz./�)rorp���r�r�)rrcr�geteuid�getegidrrrryrYrn�follow_symlinksrKrLr>�replacer�isfiler�existsrr!rF�splitZurlencoding_moderrE)r�Z	local_urir��	with_dirsrurvZ
local_basergrh�single_filer=r%�filesZrel_rootZ
iter_elements�elementsrpriror�)r?�is_srcr*r+�_get_filelist_local
sv


z-fetch_local_list.<locals>._get_filelist_localcs��jr�t�j�dkr�|j�tr*|j�}n|j�}x:|D]2}|jdd�}|sNq8|j|d||d|d�q8W|j	�|j
�j�dS)Nrrtrsrrrq)�
cache_fileryrYZmark_all_for_purge�PY3rmZ
itervalues�getZunmark_for_purge�purgeZsave)r��
local_listZlocal_list_val_iterZf_infort)r?r*r+�_maintain_cacheUs

z)fetch_local_list.<locals>._maintain_cacherz6Cache file not found or empty, creating/populating it.F)rMrQz.Expecting filename or directory instead of: %sz)Use --recursive to upload a directory: %sr)rrr�rrr�r
�getsize�loadrr�type�list�tuplerC�	recursiverrrr ryrR)�argsr�r�r�r�r�r�Z
local_urisr�r��arg�uriZlist_for_urirPr�r*)r�r?r�r+r�s<;K.




cs*�fdd��d��fdd�	}t��g}tdd�}t|�tttfkrH|g}|dkrV�j}x4|D],}t|�}|jdks~td	|��|j	|�q\Wd
}	|r�xP|D]H}||dd�\}
}|	|7}	x*|
D]"}|
|||<|j
||
j|��q�Wq�W�n*�x&|D�]}|j�}
t
jd|
d
d�}t|�dk�r�|\}}d|k}|t|�|d�\}
}|	|7}	x�|
D]*}tjj|
|d|
��rP|
|||<�qPWq�tj|j��}|�s�td|j���|�o�|ddk}||j�|j�|d�}|�r܈||�|||<|jd�}|�r|j
||�|	|jdd
�7}	q�Wt|�\}}|||	fS)Ncs�t��j|�}|jd�sdS|jt|dd�|ddjd�t|dd�d��y*|dd}|jd|i�td	|�Wntk
r�YnXdS)
NZheaderszcontent-lengthZetagz"'z
last-modified)rqrx�	timestampzs3cmd-attrsrxzretrieved md5=%s from headers)	rZobject_infor�r��intrar	r�KeyError)r��remote_item�responserx)r?r*r+�_get_remote_attribs�s
z.fetch_remote_list.<locals>._get_remote_attribsTcs�td|�d}tt��}|j|j�|j�|�d�}|j�}}|}|dkr�|ddkr�|d|jd�d�}td|j�|f�}t|�}t	dd	�}	d}
�x|d
D]�}|d}t
|d�}
|ddk}||kr�|r�tj|�}|j
�}d
}
t	dd	�}	n||d�}|j
�|}|�s td�q�|
t|d�|djd�|||dd|d�	|	|<d|	|dk�rr�t|�|	|�|	|d}|	j||�||
7}|
r�Pq�W|	|fS)Nz*Retrieving list of remote files for %s ...r)�prefixr��
uri_paramsr0rr�z
s3://%s/%sF)rMr�ZKeyZSizeTz-Found empty root object name on S3, ignoring.ZLastModifiedZETagz"')	rqr�rx�
object_key�object_uri_str�base_urirsrtrprVrxr�r�)rrrZbucket_listZbucket�object�rfindrryrr�r
rcr�rrra�
record_md5)Z
remote_urir�r��s3r�Zrem_base_originalZrem_baseZremote_uri_originalZrem_base_lenZrem_listZ	break_nowr�r�Zobject_sizerprSr�rx)r�r�r*r+�_get_filelist_remote�sX




z/fetch_remote_list.<locals>._get_filelist_remoteF)rMr�z Expecting S3 URI instead of '%s'r)r�z\*|\?r)�maxsplitr�r�r�z3Expecting S3 URI with a filename or --recursive: %s)r�r�r�rprxrq)Tr�)rrr�r�r�rCr�rrr r�r�r��rer�ry�globZfnmatchr
rcr�r�rR)r�Zrequire_attribsr�r�r�Zremote_urisZremote_listr�r�r�Z
objectlistZtmp_total_sizerSZuri_strZwildcard_split_resultr��restZneed_recursionrpr�rxrPr*)r�r?r�r+r�sdJ







csdd�}��fdd�}|dko$|dks,t�td�t��tdd�}i}td||�||�f�t�}�xh|j�D�]Z}	td	|	�|r�|	jd
�}
|
dkr�|	d|
d�}xP|r�||kr�|j|�y|d|j	d
dd�d�}Wq�t
k
r�w�Yq�Xq�W|	�k�rL�j�r(td
|	�||	=�|	=qry||�|||	�}Wn@tt
fk
�r|td|	�td|	�||	=�|	=wrYnX|�r�td|	�||	=�|	=n�y|j|	�}
Wntk
�r�d}
YnX|
dk	�r&|
�jk�r&�j|
�}td||	f�||	}|
|d<||d<|||	<||	=�|	=n$�j|	|
�||	||	<||	=�|	=qry|j|	�}
Wntk
�rtd}
YnX�j|
�}|dk	�r�td||	f�||	}|
|d<||d<|||	<||	=qr�j|	|
�qrWx6�j�D]*}||k�s�||k�s�||k�r܈|=�q�W|�||fS)NcSs|rdp
dS)NZremoteZlocalr*)Z	is_remoter*r*r+�__direction_str5sz*compare_filelists.<locals>.__direction_strc

sdd}|j|�}�j|�}|s$|rBtd|t|�t|�f�dSd�jkr�|jd�}|jd�}	|	dk	r�|dk	r�|	|kr�td|||	f�d}d�jk}
|
r�|dkr�d|dks�|dkr�d|dkr�d}
td	|�|
r�|d
dkr�d}
|o�|
�r`y|j|�}�j|�}Wn4ttfk
�r>td|�td|��YnX||k�r`d}td
|||f�|S)z@Return True if src_list[file] matches dst_list[file], else FalseTzE%s: does not exist in one side or the other: src_list=%s, dst_list=%sFrqNz'xfer: %s (size mismatch: src=%s dst=%s)rxrVzdisabled md5 check for %srpzIGNR: %s (disappeared)z%s: file disappeared, ignoring.z&XFER: %s (md5 mismatch: src=%s dst=%s))	r�r�boolr�rr�r_r|r)
rOZdst_lst�
src_remote�
dst_remoterQZ
attribs_matchZsrc_fileZdst_fileZsrc_sizeZdst_sizeZcompare_md5Zsrc_md5Zdst_md5)r?�dst_listr*r+�_compare8s@





(


z#compare_filelists.<locals>._compareFzVerifying attributes...)rMz)Comparing filelists (direction: %s -> %s)zCHECK: '%s'r�rrz!IGNR: '%s' (used --skip-existing)zIGNR: '%s' (disappeared)z%s: file disappeared, ignoring.z IGNR: '%s' (transfer not needed)zDST COPY src: '%s' -> '%s'rxZcopy_srczDST COPY dst: '%s' -> '%s'r�)rBrrrrrCrNr�rG�rindex�
ValueErrorZ
skip_existingr_r|rr�Zby_md5Zfind_md5_oner�)rOr�r�r�r�r�Zupdate_listZ
copy_pairsZ
src_dir_cacher�Zdir_idxrZ	same_filerxZ
copy_src_fileZsrc_itemrir*)r?r�r+r4s�0








)rr)FNF)*Z
__future__rrrrrZ	BaseUtilsrr	r
ZUtilsrrr
rZ
ExceptionsrrZloggingrrrrr\r�r�r}rZr{r�version_infor��__all__r"rrKrLrRrnrrrr*r*r*r+�<module>
s:

(	&
G"