HEX
Server: Apache
System: Linux zacp120.webway.host 4.18.0-553.50.1.lve.el8.x86_64 #1 SMP Thu Apr 17 19:10:24 UTC 2025 x86_64
User: govancoz (1003)
PHP: 8.3.26
Disabled: exec,system,passthru,shell_exec,proc_close,proc_open,dl,popen,show_source,posix_kill,posix_mkfifo,posix_getpwuid,posix_setpgid,posix_setsid,posix_setuid,posix_setgid,posix_seteuid,posix_setegid,posix_uname
Upload Files
File: //usr/local/lib/python3.7/test/__pycache__/test_robotparser.cpython-37.pyc
B

��g(�@s�ddlZddlZddlZddlZddlZddlmZddlm	Z	m
Z
Gdd�d�ZGdd�deej�Z
Gdd	�d	eej�ZGd
d�deej�ZGdd
�d
e�ZGdd�deej�ZGdd�deej�ZGdd�de�ZGdd�deej�ZGdd�deej�ZGdd�deej�ZGdd�deej�ZGdd�de�ZGdd�deej�ZGd d!�d!eej�ZGd"d#�d#eej�ZGd$d%�d%eej�ZGd&d'�d'eej�ZGd(d)�d)eej�ZGd*d+�d+e	�ZGd,d-�d-ej�Z Gd.d/�d/ej�Z!e"d0k�r�e�#�dS)1�N)�support)�BaseHTTPRequestHandler�
HTTPServerc@s<eZdZdZdZgZgZdd�Zdd�Zdd�Z	d	d
�Z
dS)�
BaseRobotTest�Ztest_robotparsercCs,t�|j���}tj��|_|j�|�dS)N)	�io�StringIO�
robots_txt�	readlines�urllib�robotparser�RobotFileParser�parser�parse)�self�lines�r�1/usr/local/lib/python3.7/test/test_robotparser.py�setUpszBaseRobotTest.setUpcCs$t|t�r|\}}||fS|j|fS)N)�
isinstance�tuple�agent)r�urlrrrr�get_agent_and_urls
zBaseRobotTest.get_agent_and_urlc
CsPxJ|jD]@}|�|�\}}|j||d��|�|j�||��WdQRXqWdS)N)rr)�goodr�subTest�
assertTruer�	can_fetch)rrrrrr�test_good_urlsszBaseRobotTest.test_good_urlsc
CsPxJ|jD]@}|�|�\}}|j||d��|�|j�||��WdQRXqWdS)N)rr)�badrr�assertFalserr)rrrrrr�
test_bad_urls!szBaseRobotTest.test_bad_urlsN)�__name__�
__module__�__qualname__r	rrrrrrr!rrrrr
src@s"eZdZdZddgZdddgZdS)�UserAgentWildcardTestz�User-agent: *
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
Disallow: /tmp/ # these will soon disappear
Disallow: /foo.html
    �/z
/test.htmlz/cyberworld/map/index.htmlz/tmp/xxxz	/foo.htmlN)r"r#r$r	rrrrrrr%(sr%c@s eZdZdZdddgZdgZdS)�CrawlDelayAndCustomAgentTestz�# robots.txt for http://www.example.com/

User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/ # This is an infinite virtual URL space

# Cybermapper knows where to go.
User-agent: cybermapper
Disallow:
    r&z
/test.html)Zcybermapperz/cyberworld/map/index.htmlz/cyberworld/map/index.htmlN)r"r#r$r	rrrrrrr'3s
r'c@seZdZdZgZdddgZdS)�RejectAllRobotsTestz(# go away
User-agent: *
Disallow: /
    z/cyberworld/map/index.htmlr&z/tmp/N)r"r#r$r	rrrrrrr(Dsr(c@seZdZdZdZdd�ZdS)�BaseRequestRateTestNc
Cs�|j}x�|j|jD]�}|�|�\}}|j||d��p|�|�|�|j�|�|�}|�||j�|jdk	r�|�|t	j
j�|�|j|jj�|�|j
|jj
�WdQRXqWdS)N)rr)rrrrr�assertEqual�crawl_delay�request_rateZassertIsInstancerr�RequestRateZrequestsZseconds)rrrrZparsed_request_raterrr�test_request_rateRs"



z%BaseRequestRateTest.test_request_rate)r"r#r$r,r+r.rrrrr)Nsr)c@seZdZdZdgZdS)�
EmptyFileTestrz/fooN)r"r#r$r	rrrrrr/jsr/c@s>eZdZdZdZej�dd�ZdZ	dgZ
ddd	d
ddd
gZdS)�CrawlDelayAndRequestRateTestz�User-agent: figtree
Crawl-delay: 3
Request-rate: 9/30
Disallow: /tmp
Disallow: /a%3cd.html
Disallow: /a%2fb.html
Disallow: /%7ejoe/index.html
    �figtree�	��)r1z	/foo.htmlz/tmpz	/tmp.htmlz/tmp/a.htmlz/a%3cd.htmlz/a%3Cd.htmlz/a%2fb.htmlz/~joe/index.htmlN)r"r#r$r	rrrr-r,r+rrrrrrr0os	
r0c@seZdZdZdS)�DifferentAgentTestzFigTree Robot libwww-perl/5.04N)r"r#r$rrrrrr5�sr5c@s*eZdZdZdgZddddddgZd	Zd
S)�InvalidRequestRateTestz�User-agent: *
Disallow: /tmp/
Disallow: /a%3Cd.html
Disallow: /a/b.html
Disallow: /%7ejoe/index.html
Crawl-delay: 3
Request-rate: 9/banana
    z/tmpz/tmp/z/tmp/a.htmlz/a%3cd.htmlz/a%3Cd.htmlz	/a/b.htmlz/%7Ejoe/index.htmlr4N)r"r#r$r	rrr+rrrrr6�s
	
r6c@seZdZdZdgZgZdS)�InvalidCrawlDelayTestz2User-Agent: *
Disallow: /.
Crawl-delay: pears
    z	/foo.htmlN)r"r#r$r	rrrrrrr7�sr7c@s eZdZdZdZdgZdgZdS)�AnotherInvalidRequestRateTestzeUser-agent: Googlebot
Allow: /folder1/myfile.html
Disallow: /folder1/
Request-rate: whale/banana
    �	Googlebotz/folder1/myfile.htmlz/folder1/anotherfile.htmlN)r"r#r$r	rrrrrrrr8�sr8c@seZdZdZdZdgZdS)�UserAgentOrderingTestzMUser-agent: Googlebot
Disallow: /

User-agent: Googlebot-Mobile
Allow: /
    r9z/something.jpgN)r"r#r$r	rrrrrrr:�s
r:c@seZdZdZdS)�UserAgentGoogleMobileTestzGooglebot-MobileN)r"r#r$rrrrrr;�sr;c@s eZdZdZdZdgZdgZdS)�GoogleURLOrderingTestzJUser-agent: Googlebot
Allow: /folder1/myfile.html
Disallow: /folder1/
    Z	googlebotz/folder1/myfile.htmlz/folder1/anotherfile.htmlN)r"r#r$r	rrrrrrrr<�sr<c@seZdZdZdgZdgZdS)�DisallowQueryStringTestz2User-agent: *
Disallow: /some/path?name=value
    z
/some/pathz/some/path?name=valueN)r"r#r$r	rrrrrrr=�sr=c@seZdZdZdgZdgZdS)�UseFirstUserAgentWildcardTestzNUser-agent: *
Disallow: /some/path

User-agent: *
Disallow: /another/path
    z
/another/pathz
/some/pathN)r"r#r$r	rrrrrrr>�sr>c@seZdZdZdgZdgZdS)�EmptyQueryStringTestz>User-agent: *
Allow: /some/path?
Disallow: /another/path?
    z/some/path?z/another/path?N)r"r#r$r	rrrrrrr?�sr?c@s0eZdZdZej�dd�ZdZddgZ	dgZ
dS)	�DefaultEntryTestzOUser-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/
    r4��r&z
/test.htmlz/cyberworld/map/index.htmlN)r"r#r$r	rrr-r,r+rrrrrrr@�s
r@c@seZdZdZdZdd�ZdS)�StringFormattingTestz�User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/ # This is an infinite virtual URL space

# Cybermapper knows where to go.
User-agent: cybermapper
Disallow: /some/path
    zzUser-agent: cybermapper
Disallow: /some/path

User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/

cCs|�t|j�|j�dS)N)r*�strr�expected_output)rrrr�test_string_formattingsz+StringFormattingTest.test_string_formattingN)r"r#r$r	rErFrrrrrC�s
rCc@seZdZdd�Zdd�ZdS)�RobotHandlercCs|�dd�dS)Ni�zForbidden access)Z
send_error)rrrr�do_GETszRobotHandler.do_GETcGsdS)Nr)r�format�argsrrr�log_messageszRobotHandler.log_messageN)r"r#r$rHrKrrrrrGsrGc@s*eZdZdd�Zdd�Zejdd��ZdS)�PasswordProtectedSiteTestCasecCsBttjdft�|_tjd|jjddid�|_d|j_	|j�
�dS)NrzHTTPServer servingZ
poll_intervalg{�G�z�?)�name�target�kwargsT)rr�HOSTrG�server�	threadingZThreadZ
serve_forever�tZdaemon�start)rrrrr#sz#PasswordProtectedSiteTestCase.setUpcCs"|j��|j��|j��dS)N)rQZshutdownrS�joinZserver_close)rrrr�tearDown0s

z&PasswordProtectedSiteTestCase.tearDowncCs\|jj}dtjdt|d�}|d}tj��}|�|�|�	�|�
|�d|��dS)Nzhttp://�:rBz/robots.txt�*)rQZserver_addressrrPrDrrr
Zset_url�readr r)rZaddrrZ
robots_urlrrrr�testPasswordProtectedSite5s

z7PasswordProtectedSiteTestCase.testPasswordProtectedSiteN)r"r#r$rrVrZreap_threadsrZrrrrrL!s
rLc@sFeZdZdZd�e�Zedd��Zdd�Zdd�Z	d	d
�Z
dd�Zd
S)�NetworkTestCasezhttp://www.pythontest.net/z{}elsewhere/robots.txtc	Cs@t�d�t�|j�� tj�|j�|_|j�	�WdQRXdS)NZnetwork)
rZrequiresZtransient_internet�base_urlrrr
r	rrY)�clsrrr�
setUpClassEs
zNetworkTestCase.setUpClasscCs$d�|j|tj�|�dsdnd�S)Nz{}{}{}rBr&r)rIr\�os�path�splitext)rr`rrrrLszNetworkTestCase.urlcCsV|�|jj�|�|jj�|�|j��d�|�|j�d��|�|j�d��dS)NrrX)r r�disallow_all�	allow_allZ
assertGreater�mtimer+r,)rrrr�
test_basicQs
zNetworkTestCase.test_basiccCs�|�|j�d|�d���|�|j�d|j��|�|j�d|�d���|�|j�d|�d���|�|j�d|�d���|�|j�d|j��dS)NrXZ	elsewhereZNutchZbrianZwebstats)rrrrr r\)rrrr�test_can_fetchXszNetworkTestCase.test_can_fetchcCsftj�|�d��}|��|�|j�|�|j�|�	|�
�d�|�|�d��|�|�
d��dS)Nzi-robot.txtrrX)rrr
rrYrrcr rbr*rdZassertIsNoner+r,)rrrrr�
test_read_404`szNetworkTestCase.test_read_404N)r"r#r$r\rIr	�classmethodr^rrerfrgrrrrr[@s
r[�__main__)$rr_rRZunittestZurllib.robotparserrZtestrZhttp.serverrrrZTestCaser%r'r(r)r/r0r5r6r7r8r:r;r<r=r>r?r@rCrGrLr[r"�mainrrrr�<module>s<





	)