HEX
Server: Apache
System: Linux zacp120.webway.host 4.18.0-553.50.1.lve.el8.x86_64 #1 SMP Thu Apr 17 19:10:24 UTC 2025 x86_64
User: govancoz (1003)
PHP: 8.3.26
Disabled: exec,system,passthru,shell_exec,proc_close,proc_open,dl,popen,show_source,posix_kill,posix_mkfifo,posix_getpwuid,posix_setpgid,posix_setsid,posix_setuid,posix_setgid,posix_seteuid,posix_setegid,posix_uname
Upload Files
File: //opt/cloudlinux/venv/lib/python3.11/site-packages/wmt/__pycache__/main.cpython-311.pyc
�

b(�h�<��h�ddlZddlZddlZddlZddlZddlZddlZddlmZm	Z	ddl
mZmZddl
mZddlmZddlmZmZmZmZmZddlmZmZmZmZddlmZdd	lmZdd
l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(ddl)m*Z*m+Z+ddl,m-Z-dd
l.m/Z/ddl0m1Z1ed��Z2d�Z3d�Z4eGd�d����Z5de6fd�Z7d�Z8d&d�Z9d&dee5fd�Z:d&d�Z;d�Z<d�Z=d�Z>d�Z?d�Z@d�ZAd �ZBd!�ZCeDd"kr�eEejF����ZGe2�Hd#eG��ee%eG��5ed$ed%��e"��ejejIe3��ejejJe4��ejK��ZLeL�MeC����ddd��dS#1swxYwYdSdS)'�N)�datetime�	timedelta)�List�Optional)�	dataclass)�or_)�ScrapeResult�DomainAlerts�setup_database�
session_scope�cleanup_old_data)�get_domains�setup_logger�save_pid_and_lock�	intersect)�init_sentry_client)�get_pkg_version)�PING_TIMEOUT_STATUS_CODE�
SENTRY_DNS�!ERROR_DOMAINS_PING_RETRY_INTERVAL�ERROR_DOMAINS_ALERT_INTERVAL�
WMT_LOCK_FILE�PING_CONNECTIONS�&LICENSE_EXPIRED_FAREWELL_LETTER_MARKER�LICENSE_CHECK_PAUSE)�Notifier�SupportedNotificationTypes)�ErrorReport)�cfg)�CloudlinuxLicenseLib�wmt_scannerc��tj��t�dt	tj������dS)NzReloading config: %s)r�reload�logger�info�str�to_dict��
sig_number�frames  ��/builddir/build/BUILDROOT/cl-web-monitoring-tool-3.1.8-1.el8.cloudlinux.x86_64/opt/cloudlinux/venv/lib/python3.11/site-packages/wmt/main.py�reload_confr,-s6���J�L�L�L�
�K�K�&��C�K�M�M�(:�(:�;�;�;�;�;�c�.�tjd��dS)zp
    Shutdown to call finally block to
    close all fds, remove lock and file
    see: save_pid_and_lock()
    rN)�sys�exitr(s  r+�shutdownr12s���H�Q�K�K�K�K�Kr-c�N�eZdZUeed<dZeeed<dZeeed<dS)�ScrapeCoroResult�urlN�
response_code�response_time_ms)	�__name__�
__module__�__qualname__r&�__annotations__r5r�intr6�r-r+r3r3;sD�������	�H�H�H�#'�M�8�C�=�'�'�'�&*��h�s�m�*�*�*�*�*r-r3�returnc��ttjdt��}	t|��}n#tt
f$r
t}YnwxYwt
d|��S)N�ping_connections�)�getattrrrr;�	TypeError�
ValueError�max)�values r+�get_connection_limitrFCs`���C�G�/�1A�B�B�E�!��E�
�
�����z�"�!�!�!� ����!�����q�%�=�=�s�2�A
�A
c�����K�|4�d{V����fd�}|�d��r|}d|td��d�z}n<|�d��rd|td��d�z}|}n
d|��}d|��}||���d{V��}|jdkr|cddd���d{V��S||���d{V��}|cddd���d{V��S#1�d{V��swxYwYdS)a
    Main 'pinger'
    1. Requests domains
     - if domain responded - keep status code
     - if no response for timeout - keep Timeout status code
     - if unreachable (ConnectionError or so) - keep 523 status code
       (same logic as go implementation)
    Nc
��,�K�tj��}	��|����4�d{V��}t|d��rt|j��n|}t||jtdtj��|z
z�����cddd���d{V��S#1�d{V��swxYwYdS#tj	j
$rt|t�dz���cYStj
j$rt|d���cYSwxYw)N��timeoutr4i�)r5r6i)r5)�time�get�hasattrr&r4r3�statusr;�
concurrent�futures�TimeoutErrorr�aiohttp�client_exceptions�ClientError)�
target_url�start�resp�	final_url�ping_timeout�sessions    ��r+�_fetchzping.<locals>._fetchVs�������I�K�K�E�
G�"�;�;�z�<�;�H�H��������D�18��u�1E�1E� U��D�H�
�
�
�:�I�+�!���),�T�T�Y�[�[�5�5H�-I�)J�)J���������������������������������������%�2�
N�
N�
N�'�
�6N�9E��9L�N�N�N�N�N�N��,�8�
G�
G�
G�(�
�#�F�F�F�F�F�F�
G���s;�B<�A!B)�B<�)
B3�3B<�6B3�7B<�<.D�,$D�Dzhttps://zhttp://��)�
startswith�lenr5)	r4rYrZ�	semaphorer[�	https_url�http_url�https_result�http_results	 ``      r+�pingrdLs�������$�$�$�$�$�$�$�$�	G�	G�	G�	G�	G�	G�(�>�>�*�%�%�	'��I� �3�s�:���'7�'7�#8�8�H�H�
�^�^�I�
&�
&�	'�"�S��Y�����%9�9�I��H�H�(�3�(�(�I�&����H�#�V�I�.�.�.�.�.�.�.�.���%��,�,��C$�$�$�$�$�$�$�$�$�$�$�$�$�$�F#�F�8�,�,�,�,�,�,�,�,���I$�$�$�$�$�$�$�$�$�$�$�$�$�$�$�$�$�$�$�$�$�$�$�$����$�$�$�$�$�$s�BC�2C�
C!�$C!c#�K�t��}|�|t|��z}t�dt	|����|D]*}tj|��st||||��V��+dS)z�
    ping_timeout: specified in config timeout time (s) for request
    semaphore: semaphore obj to handle asyncio tasks
    ping_target_domains: mostly needed for re-pinging error domains
    Nz Those domains will be pinged: %s)r�setr$�debugr&r�is_domain_ignoredrd)rYrZr_�ping_target_domains�domains�domains      r+�	executorsrl|s������m�m�G��&��3�*�+�+�+��
�L�L�3�S��\�\�B�B�B��A�A���$�V�,�,�	A��v�|�W�i�@�@�@�@�@��A�Ar-c���K�tt||||����}t|��dkrgSd�|D��}	tjtj|�|����d{V��S#tj$rjg}|D]*}|���s|����+|D]3}	|�	|�d{V�����#tj
$rY�0wxYw|cYSwxYw)Nrc�6�g|]}tj|����Sr<)�asyncio�create_task)�.0�coros  r+�
<listcomp>z scrape_sites.<locals>.<listcomp>�s#��>�>�>�4�W�
 ��
&�
&�>�>�>r-rI)�listrlr^ro�wait_for�gatherrQ�done�cancel�append�CancelledError)	�ping_site_timeout�
ping_intervalrZr_ri�
coroutines�tasks�results�tasks	         r+�scrape_sitesr��s1�����i� 1�7�I�GZ�[�[�\�\�J�
�:���!����	�>�>�:�>�>�>�E���%�g�n�e�&<�m�T�T�T�T�T�T�T�T�T�T����
�
�
�*,���	�	�D��9�9�;�;�
����
�
�
���	�	�D�
����T�z�z�z�z�z�z�*�*�*�*���)�
�
�
���
��������
���s7�'A,�,AC%�/C�
C%�C�C%�C�C%�$C%c
�n�t��}i}g}t|��5}|D]�}|�t|jd|j|j�����|�|j��|jdkr|j||j<�l|�|j����|�t���
tj�|�����
td���d���ddd��n#1swxYwYt��}|�|t|��z}t|��|z
}	t|��5}|	D]&}
|�t|
d������'	ddd��n#1swxYwY|S)	a
    - obtains all scrape coro results from asyncio tasks
    - saves ping results to ScrapeResult table
    - updates 'is_resolved' field in DomainAlerts table, in case
      error domain`s status code was changed to 200
    - returns domains with non-200 status code code
    T)�website�is_finishedr5r6r\)�is_resolvedF)�synchronize_sessionN)r�r�)rfr�addr	r4r5r6ry�queryr
�filterr��in_�update�dictr)�engine�pingedri�finished_domains�errors_domains�resolvedrZ�resultrj�unfinished_domains�
unfinisheds           r+�manage_ping_resultsr��sQ���u�u���N��H�	�v�	�	�G�'��	,�	,�F��K�K���
� �$�2�!'�!8�	���
�
�
�
� � ���,�,�,��#�s�*�*�-3�-A��v�z�*�*�����
�+�+�+�+��
�
�l�#�#�
�V�L�(�,�,�X�6�6�
7�
7�
�V�D�T�*�*�*��V�
F�
F�
F�!G�G�G�G�G�G�G�G�G�G�G����G�G�G�G�"�m�m�G��&��3�*�+�+�+���W���(8�8��	�v�	�	��'�,�	�	�J��K�K��"�!����
�
�
�
�	��������������������s$�C7D%�%D)�,D)�3*F*�*F.�1F.c	���tj��tt���z
}t	|��5}|�tj���tj�	t|�������ttj
|ktjdk����}d�|D��cddd��S#1swxYwYdS)z�
    - gets websites that must NOT be included in alert email:
          less than ERROR_DOMAINS_ALERT_INTERVAL passed or is_resolved marker was not
          changed from last alerting
    )�hoursFc��g|]	}|j��
Sr<�r��rq�rows  r+rsz%get_recent_alerts.<locals>.<listcomp>�s��8�8�8����8�8�8r-N)r�nowrrrr�r
r�r�r�rt�keysr�
alert_timer�)r��
alert_domains�repeat_intervalrZ�recently_alerteds     r+�get_recent_alertsr��s���l�n�n�y�7S�'T�'T�'T�T�O�	�v�	�	�9�'�"�=�=��)=�>�>�
�V�L�(�,�,�T�-�2D�2D�2F�2F�-G�-G�H�H���/�/�A�$�0�E�9�;�;�<�<�	�9�8�'7�8�8�8�9�9�9�9�9�9�9�9�9�9�9�9����9�9�9�9�9�9s�B%C+�+C/�2C/c	�Z�t�dtt|���������d�|���D��}t
tjtj	d|itj������|S)z�
    prepares needed error report object with error domains
    to be alerted and sends this mail
    returns alerted domains
    zAlerts will be sent for %sc
��g|]W\}}t|d�ttt	|������t|�������XS)z, )r4�code�count_errors)r�join�mapr&rfr^)rqrk�codess   r+rszalert.<locals>.<listcomp>�sh�����
�F�E�	�����3�s�C��J�J�/�/�0�0��U���	
�	
�	
���r-�error_report��target_email�
from_email�report�notification_type)
r$r%r&rtr��itemsrrr�r�r�ALERT�notify)�domains_datar�s  r+�alertr��s����K�K�,��D��*�*�,�,�-�-�.�.�0�0�0���*�/�/�1�1�
���L�
��%��>��L�
�5�:�
<�<�<�=C�F�H�H�H��r-c	�`��t||����fd�|���D��}|sOt�dt	t|�������t��dSt|��tj
��}t|��5}|�t���tj�����}d�|D��}|D]�}||vr^|�t���tj|k���t'|d������d|�t||�������	ddd��dS#1swxYwYdS)aJ
    - gets recently alerted domains (those that must not be alerted again)
      and does not include them for alerting
    - calls alerting for left domains
    - updates DomainAlerts table:
      if website was not alerted -> adds new record
      if website was alerted before -> updates alert time and is_resolved marker
    c�$��i|]\}}|�v�	||��
Sr<r<)rq�k�vr�s   �r+�
<dictcomp>z flush_alerts.<locals>.<dictcomp>�s*���\�\�\���A�!�K[�B[�B[��1�B[�B[�B[r-zDAll domains "%s" were alerted or still not resolved in last %d hoursNc��g|]	}|j��
Sr<r�r�s  r+rsz flush_alerts.<locals>.<listcomp>
s��0�0�0����0�0�0r-F)r�r�)r�r�)r�r�r$r%r&rtr�rr�rr�rr�r
�
with_entitiesr��allr�r�r�r�)	r�r��domains_to_alertr�rZ�websites�urlsrkr�s	        @r+�flush_alertsr��s����)���?�?��\�\�\�\��)<�)<�)>�)>�\�\�\�������Z���]�/�/�1�1�2�2�3�3�0�	2�	2�	2�	��	�
����
�,�.�.�C�	�v�	�	�J�'��=�=��.�.�
�]�<�/�
0�
0�
�S�U�U�	�1�0�x�0�0�0��&�	J�	J�F���~�~��
�
�l�+�+��V�L�0�F�:�;�;��V�D�C�U�C�C�C�D�D�D�D����L��C�H�H�H�I�I�I�I�
	J�J�J�J�J�J�J�J�J�J�J�J�J����J�J�J�J�J�Js�4C!F#�#F'�*F'c�0�|rtjjrdSdS)NTF)r�alert_notifications_enabled)�
error_domainss r+�should_be_repingedr�s ������<���t��5r-c��tj�t��r5t�d��tjt��dSdS)NzCloudLinux license was updated)�os�path�existsrr$r%�remover<r-r+�cleanup_farewell_letter_markerr�sM��	�w�~�~�<�=�=�:����4�5�5�5�
�	�8�9�9�9�9�9�:�:r-c�d�	tj�t��s_t�d��t
tjtj	itj������dSdS#t$rt�d��YdSwxYw)z@
    Sends farewell letter once (if it was not sent before)
    z/Going to send last email about expired license!r�z$Error while managing farewell letterN)r�r�r�rr$�warningrrr�r�r�FAREWELLr��	Exception�	exceptionr<r-r+�manage_license_farewellr� s���	A��w�~�~�D�E�E�	P��N�N�L�M�M�M�� �-��>��"<�"E�	
G�
G�
G�HN�v�x�x�x�x�x�
	P�	P���A�A�A����?�@�@�@�@�@�@�A���s�BB�$B/�.B/c
��:K�tj��}tjjdz}	t��t	|��t��}t
j|���}t
j|���4�d{V��}tj
|��}ttjj|||���d{V��}t||��}	t|	���r�t�dt#|	����	tjjdz}|t$kr�tj��|z
}
tjt)t+t$|
z
d��d�����d{V��|
t$krottjj||||	����	���d{V��}t|||	�����}t/|	|��}
nnt/||	��}
|	}n��|
rWt�d
t#t1|
���������t3||
��ng}	ddd���d{V��n#1�d{V��swxYwYn*#t4$rt�d��YnwxYw	tjjdz}tj��|z
}t)t+||z
d��d��}tj|���d{V��||krn�mns#	tjjdz}tj��|z
}t)t+||z
d��d��}tj|���d{V��||krn�mwxYw|S)aF
    Scanner logic:
    1. Scrapes domains and obtains ping results;
    2. Manage ping results (e.g: saving to DB)
    3. In case error domains found -> start re-pinging
       Re-pinging:
         - in min(ping_interval, 5 mins)
         - flush alerts if needed
    4. Sleep for ping_interval until next ping iteration
    �<)�limit)�	connectorNz8Those domains are unsuccessful: %s 
 Try to re-ping themTr�
)riz1Domains with unsuccessful status code found: "%s"zError during ping iteration!)rKrr|r�r
rFrR�TCPConnector�
ClientSessionro�	Semaphorer�rYr�r�r$r%r&r�sleep�minrDr�rrtr�r�r�)�previously_erroredr�rV�ping_interval_seconds�connections_limitr�rZr_�ping_resultr��elapsed_for_ping�ping_retry_result�retry_errorsr��elapsed�
sleep_times                r+�scrape_iterationr�0s�����
�I�K�K�E��G�1�B�6��9�&�(�(�(��� � � �0�2�2���(�/@�A�A�A�	��(�9�=�=�=�*	(�*	(�*	(�*	(�*	(�*	(�*	(���)�*;�<�<�I� ,���$�&;�W�i�!�!�������K�0���D�D�M�"�-�0�0�"
(����W�Y\�]j�Yk�Yk�l�l�l��,/�G�,A�B�,F�)�,�/P�P�P�+/�9�;�;��+>�(�%�m���$E�HX�$X�Z[� \� \�^`�a�a����������,�.O�O�O�6B� #�� 4�6K�W�V_�4A�4F�4F�4H�4H�7�7�7�1�1�1�1�1�1�-�,?� &�(9�=�;M�;M�;O�;O�,�,�L�-6�m�\�,R�,R�M�!�P�)2�2D�m�(T�(T�
�-:�*��/�2!�8��K�K� S� #�D��);�);�)=�)=�$>�$>� ?� ?�A�A�A� ���7�7�7��&(�"�'�U*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(�*	(����*	(�*	(�*	(�*	(���V�9�9�9����7�8�8�8�8�8�9����	�$'�G�$9�B�$>�!��i�k�k�E�)�G��S�!6��!@�!�D�D�b�I�I�J��-�
�+�+�+�+�+�+�+�+�+��.�.�.��
	���
	�$'�G�$9�B�$>�!��i�k�k�E�)�G��S�!6��!@�!�D�D�b�I�I�J��-�
�+�+�+�+�+�+�+�+�+��.�.�.��
	�	�	�	�	��sP�AJ�G$I=�+J�=
J�J�
J�J�L(�$J6�3L(�5J6�6L(�(A0Nc��K�t��}g}d}	t�����rd}t||���d{V��}n~|dz
}|dkrt�d��|dkrt
��d}tjjdz}t|t��}tj|���d{V����)z�
    Main loop for wmt_scanner_solo service
    each 'while: True' iteration returns errored domains
    (domains that responded with non-200 status code)
    rTNr@z)Seems your CloudLinux license is expired!�r�)
rr �get_license_statusr�r$r�r�rr|r�rror�)r�r��license_attemptr�r�s     r+�scrape_loopr�{s������
�
�F����O�,��!�!�4�4�6�6�	,��O�'7�8J�F�'S�'S�!S�!S�!S�!S�!S�!S����q� �O��!�#�#����J�K�K�K��!�#�#�'�)�)�)�"#��$'�G�$9�B�$>�!��2�4G�H�H�J��-�
�+�+�+�+�+�+�+�+�+�,r-�__main__zPID: %szweb-monitoring-toolzcl-web-monitoring-tool)N)Nror�r/rR�signalrK�concurrent.futuresrOrr�typingrr�dataclassesr�
sqlalchemyr�wmt.dbr	r
rrr
�wmt.common.utilsrrrr�clsentryr�clsentry.utilsr�wmt.common.constrrrrrrrr�wmt.common.notificationrr�wmt.common.reportr�
wmt.commonr�	cllicenser r$r,r1r3r;rFrdrlr�r�r�r�r�r�r�r�r�r�r7r&�getpid�pidr%�SIGUSR1�SIGTERM�get_event_loop�loop�run_until_completer<r-r+�<module>rs�������	�	�	�	�
�
�
�
�����
�
�
�
���������(�(�(�(�(�(�(�(�!�!�!�!�!�!�!�!�!�!�!�!�!�!���������������������U�T�T�T�T�T�T�T�T�T�T�T�'�'�'�'�'�'�*�*�*�*�*�*�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�I�H�H�H�H�H�H�H�)�)�)�)�)�)�������*�*�*�*�*�*�	��m�	$�	$��<�<�<�
����+�+�+�+�+�+�+���+��c�����-�-�-�`A�A�A�A���jn�o�kA�����('�'�'�'�T9�9�9����4J�J�J�D���:�:�:�
A�
A�
A� H�H�H�V,�,�,�6�z���

�#�i�b�i�k�k�
�
�C�
�K�K�	�3����	�	�=�#�	.�	.�/�/���0�*�?�+C�D�D�%�	'�	'�	'�	��
�f�n�k�2�2�2���
�f�n�h�/�/�/�%�w�%�'�'�������
�
�.�.�.�/�/�/�/�/�/�/�/�/�/�/�/����/�/�/�/�/�/��s�)A/F%�%F)�,F)