File: //usr/local/lib/python3.7/test/__pycache__/test_robotparser.cpython-37.pyc
B
��g( � @ s� d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm Z m
Z
G dd� d�Z G dd� de ej �Z
G dd � d e ej �ZG d
d � d e ej �ZG d d
� d
e �ZG dd� deej �ZG dd� deej �ZG dd� de�ZG dd� de ej �ZG dd� de ej �ZG dd� de ej �ZG dd� de ej �ZG d d � d e�ZG d d � d e ej �ZG d d!� d!e ej �ZG d"d#� d#e ej �ZG d$d%� d%e ej �Z G d&d'� d'eej �Z G d(d)� d)e ej �Z G d*d+� d+e �Z G d,d-� d-ej �Z G d.d/� d/ej �Z!e"d0k�r�e�#� dS )1� N)�support)�BaseHTTPRequestHandler�
HTTPServerc @ s< e Zd ZdZdZg Zg Zdd� Zdd� Zdd� Z d d
� Z
d S ) �
BaseRobotTest� Ztest_robotparserc C s, t �| j��� }tj�� | _| j�|� d S )N) �io�StringIO�
robots_txt� readlines�urllib� robotparser�RobotFileParser�parser�parse)�self�lines� r �1/usr/local/lib/python3.7/test/test_robotparser.py�setUp s zBaseRobotTest.setUpc C s$ t |t�r|\}}||fS | j|fS )N)�
isinstance�tuple�agent)r �urlr r r r �get_agent_and_url s
z BaseRobotTest.get_agent_and_urlc
C sP xJ| j D ]@}| �|�\}}| j||d�� | �| j�||�� W d Q R X qW d S )N)r r )�goodr �subTest�
assertTruer � can_fetch)r r r r r r �test_good_urls s z BaseRobotTest.test_good_urlsc
C sP xJ| j D ]@}| �|�\}}| j||d�� | �| j�||�� W d Q R X qW d S )N)r r )�badr r � assertFalser r )r r r r r r �
test_bad_urls! s zBaseRobotTest.test_bad_urlsN) �__name__�
__module__� __qualname__r r r r r r r r! r r r r r
s r c @ s" e Zd ZdZddgZdddgZdS )�UserAgentWildcardTestz�User-agent: *
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
Disallow: /tmp/ # these will soon disappear
Disallow: /foo.html
�/z
/test.htmlz/cyberworld/map/index.htmlz/tmp/xxxz /foo.htmlN)r" r# r$ r r r r r r r r% ( s r% c @ s e Zd ZdZdddgZdgZdS )� CrawlDelayAndCustomAgentTestz�# robots.txt for http://www.example.com/
User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
# Cybermapper knows where to go.
User-agent: cybermapper
Disallow:
r& z
/test.html)Z cybermapperz/cyberworld/map/index.htmlz/cyberworld/map/index.htmlN)r" r# r$ r r r r r r r r' 3 s
r' c @ s e Zd ZdZg ZdddgZdS )�RejectAllRobotsTestz(# go away
User-agent: *
Disallow: /
z/cyberworld/map/index.htmlr& z/tmp/N)r" r# r$ r r r r r r r r( D s r( c @ s e Zd ZdZdZdd� ZdS )�BaseRequestRateTestNc
C s� | j }x�| j| j D ]�}| �|�\}}| j||d��p | �|�|�| j� |�|�}| �|| j� | jd k r�| �|t j
j � | �|j | jj � | �|j
| jj
� W d Q R X qW d S )N)r r )r r r r r � assertEqual� crawl_delay� request_rateZassertIsInstancer r � RequestRateZrequestsZseconds)r r r r Zparsed_request_rater r r �test_request_rateR s"
z%BaseRequestRateTest.test_request_rate)r" r# r$ r, r+ r. r r r r r) N s r) c @ s e Zd ZdZdgZdS )�
EmptyFileTestr z/fooN)r" r# r$ r r r r r r r/ j s r/ c @ s> e Zd ZdZdZej�dd�ZdZ dgZ
ddd d
d d d
gZ dS )� CrawlDelayAndRequestRateTestz�User-agent: figtree
Crawl-delay: 3
Request-rate: 9/30
Disallow: /tmp
Disallow: /a%3cd.html
Disallow: /a%2fb.html
Disallow: /%7ejoe/index.html
�figtree� � � )r1 z /foo.htmlz/tmpz /tmp.htmlz /tmp/a.htmlz /a%3cd.htmlz /a%3Cd.htmlz /a%2fb.htmlz/~joe/index.htmlN) r" r# r$ r r r r r- r, r+ r r r r r r r0 o s
r0 c @ s e Zd ZdZdS )�DifferentAgentTestz FigTree Robot libwww-perl/5.04N)r" r# r$ r r r r r r5 � s r5 c @ s* e Zd ZdZdgZddddddgZd Zd
S ) �InvalidRequestRateTestz�User-agent: *
Disallow: /tmp/
Disallow: /a%3Cd.html
Disallow: /a/b.html
Disallow: /%7ejoe/index.html
Crawl-delay: 3
Request-rate: 9/banana
z/tmpz/tmp/z /tmp/a.htmlz /a%3cd.htmlz /a%3Cd.htmlz /a/b.htmlz/%7Ejoe/index.htmlr4 N)r" r# r$ r r r r+ r r r r r6 � s
r6 c @ s e Zd ZdZdgZg ZdS )�InvalidCrawlDelayTestz2User-Agent: *
Disallow: /.
Crawl-delay: pears
z /foo.htmlN)r" r# r$ r r r r r r r r7 � s r7 c @ s e Zd ZdZdZdgZdgZdS )� AnotherInvalidRequestRateTestzeUser-agent: Googlebot
Allow: /folder1/myfile.html
Disallow: /folder1/
Request-rate: whale/banana
� Googlebotz/folder1/myfile.htmlz/folder1/anotherfile.htmlN)r" r# r$ r r r r r r r r r8 � s r8 c @ s e Zd ZdZdZdgZdS )�UserAgentOrderingTestzMUser-agent: Googlebot
Disallow: /
User-agent: Googlebot-Mobile
Allow: /
r9 z/something.jpgN)r" r# r$ r r r r r r r r: � s
r: c @ s e Zd ZdZdS )�UserAgentGoogleMobileTestzGooglebot-MobileN)r" r# r$ r r r r r r; � s r; c @ s e Zd ZdZdZdgZdgZdS )�GoogleURLOrderingTestzJUser-agent: Googlebot
Allow: /folder1/myfile.html
Disallow: /folder1/
Z googlebotz/folder1/myfile.htmlz/folder1/anotherfile.htmlN)r" r# r$ r r r r r r r r r< � s r< c @ s e Zd ZdZdgZdgZdS )�DisallowQueryStringTestz2User-agent: *
Disallow: /some/path?name=value
z
/some/pathz/some/path?name=valueN)r" r# r$ r r r r r r r r= � s r= c @ s e Zd ZdZdgZdgZdS )� UseFirstUserAgentWildcardTestzNUser-agent: *
Disallow: /some/path
User-agent: *
Disallow: /another/path
z
/another/pathz
/some/pathN)r" r# r$ r r r r r r r r> � s r> c @ s e Zd ZdZdgZdgZdS )�EmptyQueryStringTestz>User-agent: *
Allow: /some/path?
Disallow: /another/path?
z /some/path?z/another/path?N)r" r# r$ r r r r r r r r? � s r? c @ s0 e Zd ZdZej�dd�ZdZddgZ dgZ
dS ) �DefaultEntryTestzOUser-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/
r4 � � r& z
/test.htmlz/cyberworld/map/index.htmlN) r" r# r$ r r r r- r, r+ r r r r r r r@ � s
r@ c @ s e Zd ZdZdZdd� ZdS )�StringFormattingTestz�User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
# Cybermapper knows where to go.
User-agent: cybermapper
Disallow: /some/path
zzUser-agent: cybermapper
Disallow: /some/path
User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/
c C s | � t| j�| j� d S )N)r* �strr �expected_output)r r r r �test_string_formatting s z+StringFormattingTest.test_string_formattingN)r" r# r$ r rE rF r r r r rC � s
rC c @ s e Zd Zdd� Zdd� ZdS )� RobotHandlerc C s | � dd� d S )Ni� zForbidden access)Z
send_error)r r r r �do_GET s zRobotHandler.do_GETc G s d S )Nr )r �format�argsr r r � log_message s zRobotHandler.log_messageN)r" r# r$ rH rK r r r r rG s rG c @ s* e Zd Zdd� Zdd� Zejdd� �ZdS )� PasswordProtectedSiteTestCasec C sB t tjdft�| _tjd| jjddid�| _d| j_ | j�
� d S )Nr zHTTPServer servingZ
poll_intervalg{�G�z�?)�name�target�kwargsT) r r �HOSTrG �server� threadingZThreadZ
serve_forever�tZdaemon�start)r r r r r # s z#PasswordProtectedSiteTestCase.setUpc C s" | j �� | j�� | j �� d S )N)rQ ZshutdownrS �joinZ server_close)r r r r �tearDown0 s
z&PasswordProtectedSiteTestCase.tearDownc C s\ | j j}dtj d t|d � }|d }tj�� }|�|� |� � | �
|� d|�� d S )Nzhttp://�:rB z /robots.txt�*) rQ Zserver_addressr rP rD r r r
Zset_url�readr r )r Zaddrr Z
robots_urlr r r r �testPasswordProtectedSite5 s
z7PasswordProtectedSiteTestCase.testPasswordProtectedSiteN)r" r# r$ r rV r Z reap_threadsrZ r r r r rL ! s
rL c @ sF e Zd ZdZd�e�Zedd� �Zdd� Zdd� Z d d
� Z
d d � Z d
S )�NetworkTestCasezhttp://www.pythontest.net/z{}elsewhere/robots.txtc C s@ t �d� t �| j�� tj�| j�| _| j� � W d Q R X d S )NZnetwork)
r ZrequiresZtransient_internet�base_urlr r r
r r rY )�clsr r r �
setUpClassE s
zNetworkTestCase.setUpClassc C s$ d� | j|tj�|�d s dnd�S )Nz{}{}{}rB r& r )rI r\ �os�path�splitext)r r` r r r r L s zNetworkTestCase.urlc C sV | � | jj� | � | jj� | �| j�� d� | � | j�d�� | � | j�d�� d S )Nr rX )r r � disallow_all� allow_allZ
assertGreater�mtimer+ r, )r r r r �
test_basicQ s
zNetworkTestCase.test_basicc C s� | � | j�d| �d��� | �| j�d| j�� | �| j�d| �d��� | �| j�d| �d��� | �| j�d| �d��� | � | j�d| j�� d S )NrX Z elsewhereZNutchZbrianZwebstats)r r r r r r\ )r r r r �test_can_fetchX s z NetworkTestCase.test_can_fetchc C sf t j�| �d��}|�� | �|j� | �|j� | � |�
� d� | � |� d�� | � |�
d�� d S )Nz i-robot.txtr rX )r r r
r rY r rc r rb r* rd Z assertIsNoner+ r, )r r r r r �
test_read_404` s z NetworkTestCase.test_read_404N) r" r# r$ r\ rI r � classmethodr^ r re rf rg r r r r r[ @ s
r[ �__main__)$r r_ rR ZunittestZurllib.robotparserr Ztestr Z http.serverr r r ZTestCaser% r' r( r) r/ r0 r5 r6 r7 r8 r: r; r< r= r> r? r@ rC rG rL r[ r" �mainr r r r �<module> s<
)