He%5PddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z m Z ddl mZddlmZddlmZmZmZmZmZddlmZmZmZmZddlmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(dd l)m*Z*m+Z+dd l,m-Z-dd l.m/Z/ddl0m1Z1edZ2dZ3dZ4eGddZ5dZ6d%dZ7d%de fdZ8d%dZ9dZ:dZ;dZdZ?dZ@d ZAeBd!kreCejDZEe2Fd"eEee%eE5ed#ed$e"ejejGe3ejejHe4ejIZJeJKeAddddS#1swxYwYdSdS)&N)datetime timedelta)OptionalIterable) dataclass)or_) ScrapeResult DomainAlertssetup_database session_scopecleanup_old_data) get_domains setup_loggersave_pid_and_lock intersect)init_sentry_client)get_pkg_version)PING_TIMEOUT_STATUS_CODE SENTRY_DNS!ERROR_DOMAINS_PING_RETRY_INTERVALERROR_DOMAINS_ALERT_INTERVAL WMT_LOCK_FILEPING_CONNECTIONS&LICENSE_EXPIRED_FAREWELL_LETTER_MARKERLICENSE_CHECK_PAUSE)NotifierSupportedNotificationTypes) ErrorReport)cfg)CloudlinuxLicenseLib wmt_scannerctjtdt tjdS)NzReloading config: %s)rreloadloggerinfostrto_dict sig_numberframes =/opt/cloudlinux/venv/lib/python3.11/site-packages/wmt/main.py reload_confr,-s6JLLL KK&CKMM(:(:;;;;;c.tjddS)zp Shutdown to call finally block to close all fds, remove lock and file see: save_pid_and_lock() rN)sysexitr(s r+shutdownr12s HQKKKKKr-cNeZdZUeed<dZeeed<dZeeed<dS)ScrapeCoroResulturlN response_coderesponse_time_ms) __name__ __module__ __qualname__r&__annotations__r5rintr6r-r+r3r3;sD HHH#'M8C='''&*hsm*****r-r3c Ktj4d{V}|4d{Vtj} |||4d{V}t ||jt dtj|z zcdddd{Vcdddd{Vcdddd{VS#1d{VswxYwYn#tjj $rAt |t|dzcYcdddd{Vcdddd{VStj j $r8t |dcYcdddd{Vcdddd{VSwxYw dddd{Vn#1d{VswxYwYdddd{VdS#1d{VswxYwYdS)a Main 'pinger' 1. Requests domains - if domain responded - keep status code - if no response for timeout - keep Timeout status code - if unreachable (ConnectionError or so) - keep 523 status code (same logic as go implementation) Ntimeouti)r5r6i )r5) aiohttp ClientSessiontimegetr3statusr; concurrentfutures TimeoutErrorrclient_exceptions ClientError)r4 ping_timeout semaphoresessionstartresps r+pingrOCs$&&<<<<<<<'9<<<<<<<<  <{{3 {==       't{%(u1D)E%F%F             <<<<<<<<<<<<<<<<<<<<<<<<<<<                !. J J J#C2J5AD5HJJJ J J<<<<<<<<<<<<<<<<<<<<<<<<<<<(4 < < <$Cs;;; ; ;!<<<<<<<<<<<<<<<<<<<<<<<<<<< < <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<F4 F F4F F44 F>F>c#K|tn|}|D])}tj|st|||V*dS)z ping_timeout: specified in config timeout time (s) for request semaphore: semaphore obj to handle asyncio tasks ping_target_domains: mostly needed for re-pinging error domains N)rris_domain_ignoredrO)rJrKping_target_domainsdomainsdomains r+ executorsrU_sc 3:kmmm  88$V,, 8v|Y77 7 7 788r-returncKdt|||D}t|dkrgStjt ||d{V\}}|S)Nc6g|]}tj|Sr<)asyncio create_task).0 coroutines r+ z scrape_sites..ms9 \ \ \  + + \ \ \r-rr>)rUlenrYwaittuple)ping_site_timeout ping_intervalrKrRtaskspinged_s r+ scrape_sitesrfls \ \'(99FYZZ \ \ \E 5zzQ l5<<GGGGGGGGGIFA Mr-c vt}i}g}t|5}|D]}|}|t |jd|j|j||j|jdkr|j||j<||j| t tj |tdddddn #1swxYwY||n t!} t| |z } t|5}| D]&} |t | d' dddn #1swxYwY|S) a - obtains all scrape coro results from asyncio tasks - saves ping results to ScrapeResult table - updates 'is_resolved' field in DomainAlerts table, in case error domain`s status code was changed to 200 - returns domains with non-200 status code code T)website is_finishedr5r6) is_resolvedF)synchronize_sessionN)rhri)setr resultaddr r4r5r6appendqueryr filterrhin_updatedictr) enginerdrRfinished_domainserrors_domainsresolvedrLtaskrnrSunfinished_domains unfinisheds r+manage_ping_resultsr}usTuuNH v  G' , ,D[[]]F KK   $2!'!8       , , ,#s**-3-Avz** ++++ l## VL(,,X66 7 7 VDT***V F F F#GGGGGGGGGGGGGGG$&9%D!! ]] W(88 v  ',  J KK "!      s$D D99D=D=7*F..F25F2c tjttz }t |5}|t jt j t| tt j |kt jdk}d|DcdddS#1swxYwYdS)z - gets websites that must NOT be included in alert email: less than ERROR_DOMAINS_ALERT_INTERVAL passed or is_resolved marker was not changed from last alerting )hoursFcg|] }|j Sr<rhr[rows r+r]z%get_recent_alerts..s888 888r-N)rnowrrr rqr rhrrrslistkeysr alert_timerk)rv alert_domainsrepeat_intervalrLrecently_alerteds r+get_recent_alertsrs lnny7S'T'T'TTO v  9'"==)=>> VL(,,T-2D2D2F2F-G-GHH //A$0E9;;<< 98'7888 999999999999999999sB%C++C/2C/c Ztdtt|d|D}t tjtj d|itj  |S)z prepares needed error report object with error domains to be alerted and sends this mail returns alerted domains zAlerts will be sent for %sc g|]W\}}t|dttt |t |XS)z, )r4code count_errors)rjoinmapr&rmr^)r[rTcodess r+r]zalert..sh FE 3sCJJ//00U   r- error_report target_email from_emailreportnotification_type) r$r%r&rritemsrrrrrALERTnotify) domains_datars r+alertrs  KK,D**,,--..000 *//11 L %> L 5: <<< =CFHHH r-c `t||fd|D}|sOtdt t |tdSt|tj }t|5}| ttj}d|D}|D]}||vr^| ttj|kt'|dd|t|| ddddS#1swxYwYdS)aJ - gets recently alerted domains (those that must not be alerted again) and does not include them for alerting - calls alerting for left domains - updates DomainAlerts table: if website was not alerted -> adds new record if website was alerted before -> updates alert time and is_resolved marker c$i|] \}}|v || Sr<r<)r[kvrs r+ z flush_alerts..s*\\\A!K[B[B[1B[B[B[r-zDAll domains "%s" were alerted or still not resolved in last %d hoursNcg|] }|j Sr<rrs r+r]z flush_alerts..s000 000r-F)rrk)rhr)rrr$r%r&rrrrrrr rqr with_entitiesrhallrrrtruro) rvrdomains_to_alertrrLwebsitesurlsrTrs @r+ flush_alertsrs)??\\\\)<)<)>)>\\\  Z]//1122330 2 2 2   ,..C v   J'==.. ]</ 0 0 SUU 10x000& J JF~~ l++VL0F:;;VDCUCCCDDDD LCHHHIIII  J J J J J J J J J J J J J J J J J J Js4C!F##F'*F'c0|rtjjrdSdS)NTF)ralert_notifications_enabled) error_domainss r+should_be_repingedrs <t 5r-ctjtr5tdtjtdSdS)NzCloudLinux license was updated)ospathexistsrr$r%remover<r-r+cleanup_farewell_letter_markerrsM w~~<==: 4555 899999::r-cd tjts_tdt tjtj itj  dSdS#t$rtdYdSwxYw)z@ Sends farewell letter once (if it was not sent before) z/Going to send last email about expired license!rz$Error while managing farewell letterN)rrrrr$warningrrrrrFAREWELLr Exception exceptionr<r-r+manage_license_farewellrs Aw~~DEE P NNL M M M  ->"<"E  G G GHNvxxxxx  P P AAA?@@@@@@AsBB$B/.B/c zKtj}tjjdz} tt |t jt}ttjj ||d{V}t||}t|rt dt| tjjdz}|tkrtj|z }t jt#t%t|z ddd{V|tkrnttjj |||d{V}t|||} t)|| } nnt)||} |}n| rWt dtt+| t-|| ng}n*#t.$rtd YnwxYw tjjdz}tj|z } t#t%|| z dd} t j| d{V| |krnmns# tjjdz}tj|z } t#t%|| z dd} t j| d{V| |krnmwxYw|S) aF Scanner logic: 1. Scrapes domains and obtains ping results; 2. Manage ping results (e.g: saving to DB) 3. In case error domains found -> start re-pinging Re-pinging: - in min(ping_interval, 5 mins) - flush alerts if needed 4. Sleep for ping_interval until next ping iteration <Nz8Those domains are unsuccessful: %s Try to re-ping themTr )rRz1Domains with unsuccessful status code found: "%s"zError during ping iteration!)rBrrbrr rY SemaphorerrfrJr}rr$r%r&rsleepminmaxrrrrrr) previously_erroredrvrMping_interval_secondsrK ping_resultrelapsed_for_pingping_retry_result retry_errorsrelapsed sleep_times r+scrape_iterationrs IKKEG1B62&(((   %&677 ()=?TV_```````` +FK@@  m , , $ KKSUXYfUgUg h h h (+(=(B%(+LLL'+y{{U':$!-C ADT TVWXXZ\]](*KKK2>G02G0=0B0B0D0D333------)(;6CTVcVhVhVjVj'k'k (1-(N(N L%..@-$P$PM)6&+ . 4 O]%7%7%9%9 : :;;===V]333"$  9997888889 $'G$9B$> !ikkE)GS!6!@!DDbIIJ- ++ + + + + + + +...     $'G$9B$> !ikkE)GS!6!@!DDbIIJ- ++ + + + + + + +...       s+HH/.K/$IKIKA0L8cKt}g}d} trd}t||d{V}n~|dz }|dkrtd|dkrt d}tjjdz}t|t}tj |d{V)z Main loop for wmt_scanner_solo service each 'while: True' iteration returns errored domains (domains that responded with non-200 status code) rTNz)Seems your CloudLinux license is expired!r) r r get_license_statusrr$rrrrbrrrYr)rvrlicense_attemptrrs r+ scrape_looprJs   FO,  ! ! 4 4 6 6 ,O'78JF'S'S!S!S!S!S!S!S   q O!##JKKK!##')))"#$'G$9B$> !24GHHJ- ++ + + + + + + +,r-__main__zPID: %szweb-monitoring-toolzcl-web-monitoring-tool)N)LrYrr/r@signalrBconcurrent.futuresrErrtypingrr dataclassesr sqlalchemyrwmt.dbr r r r r wmt.common.utilsrrrrclsentryrclsentry.utilsrwmt.common.constrrrrrrrrwmt.common.notificationrrwmt.common.reportr wmt.commonr cllicenser r$r,r1r3rOrUrfr}rrrrrrrrr7r&getpidpidr%SIGUSR1SIGTERMget_event_looplooprun_until_completer<r-r+rsX  ((((((((%%%%%%%%!!!!!!UTTTTTTTTTTT''''''******                    IHHHHHHH))))))****** m $ $<<<  +++++++ +<<<8 8 8 8 8ai''''T 9 9 94JJJD ::: A A A AAAH,,,6 z #ibikk  C KK 3  =# . .//0*?+CDD% ' ' '  fnk222 fnh///%w%''  ...//////////////////sA/FF F