File: //usr/local/lib/python3.10/urllib/__pycache__/robotparser.cpython-310.opt-2.pyc
o
�i�$ � @ sZ d dl Z d dlZd dlZdgZe �dd�ZG dd� d�ZG dd� d�ZG dd � d �Z dS )
� N�RobotFileParser� RequestRatezrequests secondsc @ sp e Zd Z ddd�Zdd� Zdd� Zdd � Zd
d � Zd d
� Zdd� Z dd� Z
dd� Z dd� Z dd� Z
dd� ZdS ) r � c C s2 g | _ g | _d | _d| _d| _| �|� d| _d S )NFr )�entries�sitemaps�
default_entry� disallow_all� allow_all�set_url� last_checked��self�url� r �//usr/local/lib/python3.10/urllib/robotparser.py�__init__ s
zRobotFileParser.__init__c C s | j S �N)r �r
r r r �mtime% s zRobotFileParser.mtimec C s dd l }|� � | _d S )Nr )�timer )r
r r r r �modified. s zRobotFileParser.modifiedc C s( || _ tj�|�dd� \| _| _d S )N� � )r �urllib�parse�urlparse�host�pathr r r r r
6 s zRobotFileParser.set_urlc
C s� z t j�| j�}W n@ t jjyJ } z2|jdv r d| _n|jdkr1|jdk r?d| _W Y d }~d S W Y d }~d S W Y d }~d S W Y d }~d S d }~ww |� � }| �
|� d�� � � d S )N)i� i� Ti� i� zutf-8)
r Zrequest�urlopenr �error� HTTPError�coder r �readr �decode�
splitlines)r
�f�err�rawr r r r" ; s
����zRobotFileParser.readc C s2 d|j v r| jd u r|| _d S d S | j�|� d S �N�*)�
useragentsr r �append)r
�entryr r r �
_add_entryH s
�zRobotFileParser._add_entryc C sL d}t � }| �� |D �]
}|s)|dkrt � }d}n|dkr)| �|� t � }d}|�d�}|dkr8|d |� }|�� }|s?q |�dd�}t|�dk�r|d �� �� |d< tj �
|d �� �|d< |d dkr|dkrt| �|� t � }|j � |d � d}q |d dkr�|dkr�|j
� t|d d�� d}q |d d kr�|dkr�|j
� t|d d
�� d}q |d d kr�|dkr�|d �� �� r�t|d �|_d}q |d d k�r|dk�r|d �d
�}t|�dk�r|d �� �� �r|d �� �� �rtt|d �t|d ��|_d}q |d dk�r| j� |d � q |dk�r$| �|� d S d S )Nr r � �#�:z
user-agentZdisallowFZallowTz crawl-delayz request-rate�/Zsitemap)�Entryr r- �find�strip�split�len�lowerr r �unquoter* r+ � rulelines�RuleLine�isdigit�int�delayr �req_rater )r
�lines�stater, �line�iZnumbersr r r r Q sx
� � �
� ��
�zRobotFileParser.parsec C s� | j rdS | jr dS | jsdS tj�tj�|��}tj�dd|j|j |j
|j f�}tj� |�}|s4d}| j
D ]}|�|�rE|�|� S q7| jrO| j�|�S dS )NFTr r1 )r r r r r r r8 �
urlunparser �params�query�fragment�quoter �
applies_to� allowancer )r
� useragentr Z
parsed_urlr, r r r � can_fetch� s* �
� zRobotFileParser.can_fetchc C �> | � � sd S | jD ] }|�|�r|j S q | jr | jjS d S r )r r rH r= r �r
rJ r, r r r � crawl_delay� �
�zRobotFileParser.crawl_delayc C rL r )r r rH r> r rM r r r � request_rate� rO z RobotFileParser.request_ratec C s | j sd S | j S r )r r r r r � site_maps� s zRobotFileParser.site_mapsc C s, | j }| jd ur|| jg }d�tt|��S )Nz
)r r �join�map�str)r
r r r r �__str__� s
zRobotFileParser.__str__N)r )�__name__�
__module__� __qualname__r r r r
r" r- r rK rN rP rQ rU r r r r r s
I
c @ s& e Zd Z dd� Zdd� Zdd� ZdS )r: c C s< |dkr|sd}t j�t j�|��}t j�|�| _|| _d S )Nr T)r r rC r rG r rI )r
r rI r r r r � s
zRuleLine.__init__c C s | j dkp
|�| j �S r( )r �
startswith)r
�filenamer r r rH � s zRuleLine.applies_toc C s | j rdndd | j S )NZAllowZDisallowz: )rI r r r r r rU � s zRuleLine.__str__N)rV rW rX r rH rU r r r r r: � s
r: c @ s. e Zd Z dd� Zdd� Zdd� Zdd� Zd S )
r2 c C s g | _ g | _d | _d | _d S r )r* r9 r= r> r r r r r � s
zEntry.__init__c C s� g }| j D ]
}|�d|� �� q| jd ur |�d| j� �� | jd ur3| j}|�d|j� d|j� �� |�tt| j �� d�
|�S )Nz User-agent: z
Crawl-delay: zRequest-rate: r1 �
) r* r+ r= r> ZrequestsZseconds�extendrS rT r9 rR )r
�ret�agentZrater r r rU � s
z
Entry.__str__c C sH |� d�d �� }| jD ]}|dkr dS |�� }||v r! dS q
dS )Nr1 r r) TF)r5 r7 r* )r
rJ r^ r r r rH � s
�zEntry.applies_toc C s& | j D ] }|�|�r|j S qdS )NT)r9 rH rI )r
rZ rA r r r rI
s
�zEntry.allowanceN)rV rW rX r rU rH rI r r r r r2 � s
r2 )
� collections� urllib.parser Zurllib.request�__all__�
namedtupler r r: r2 r r r r �<module> s B