3 `_@sddlmZddlZddlZejdkrzyddlmZWnek rNeZYnXddl Z ddl Z ddl m Z mZddlZn(ddlZddlZddlZddl m Z mZddlZddlZy ddlZWnek rddlZYnXddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z y0ddl!m"Z"m#Z#ddl$m%Z%m&Z&ddl'm(Z(WnBek rdd l"m"Z"dd l m#Z#dd l)m%Z%m&Z&m(Z(YnXd Z*ej+j,ej+j-e.Z/ej+j0d dej+j0d dej+j0ej+j,e/dej+j0ej+j,e/dej+j-dej+j-dfZ1ej2dkr8ej+j0e/dej+j0e/dfZ1ddZ3d ddZ4ddZ5gfddZ6GdddZ7ddZ8e9dkre8dS)!)print_functionN) basestring)QueueEmpty)urlparse urlencode)urlopenRequest) HTTPError)r)r)r r r z1.2.17z/etcz nixstats.iniznixstats-token.iniwin32cCsVtdd}|jjdd}|jdd}djdtd d j|d |d |jjdd fS)z Return string with info about nixstatsagent: - version - plugins enabled - absolute path to plugin directory - server id from configuration file T) dry_instanceagentpluginsenabled)state z Version: %szPlugins enabled: %sz, zPlugins directory: %sz Server: %sserver)Agentconfigget _get_pluginsjoin __version__)r plugins_pathZplugins_enabledrE/usr/local/lib/python3.6/site-packages/nixstatsagent/nixstatsagent.pyinfoBs   rhttpscCs.tjd}tdd}ttjdkr.tjd}ntjjtd}ttjdkrVtjd}nd}d|kr|jdd}|jdd }njytj d}Wnt k rt j }YnXt |d |jjd d d t|||djddjj}t|dkrtd|t|djd||fn td|dS)NT)r znixstats-token.inir_rz://dataapi_hostz /hello.php)userhostname unique_idzutf-8)r#zGot server_id: %swz[DEFAULT] user=%s server=%s z Could not retrieve server_id: %s)sysargvrlenospathr__FILEABSDIRNAME__splitunameAttributeErrorsocketgetfqdnr rrrencodereaddecodeprintopenwrite)protoZuser_idrZtoken_filenamer'Z server_idr&rrrhelloWs6       r<cCs0t|tr&tjj|}tjj|dS|jSdS)Nr) isinstancerr-r.basenamesplitext__name__)pluginr>rrr _plugin_name~s  rBc 3Csjtdd}|jjdd}|tjkr0tjjd||sR|jdd}tdd j|x|D]}td |yt j |\}}}Wn.t k r}ztd |wZWYd d }~XnXzHyt j ||||}Wn.t k r}ztd |wZWYd d }~XnXWd |r|j Xy(|jj|j} ttj| dddWqZt k r`}ztd|WYd d }~XqZXqZWd S)z Test specified plugins and print their data output after single check. If plugins list is empty test all enabled plugins. T)r rrrr)rzCheck all enabled plugins: %sz, z%s:z Find error:Nz Load error:)indent sort_keyszExecution error:)rrrr*r.insertrr8rimp find_module Exception load_moduleclosePluginrunjsondumps) rrr plugin_namefppathname descriptionemodulepayloadrrr test_pluginss4       rWc@seZdZeZeZeZeZdZdddZ ddZ ddZ dd Z d d Z d d ZddZddZddZddZdddZddZddZdS) rFcCs:|ji|_|rdS|j|j|j|jdS)z0 Initialize internal strictures N) _config_init plugins_cache _logging_init _plugins_init_data_worker_init _dump_config)selfr rrr__init__szAgent.__init__cCsddtjdddtjjtdddddddd d dd }d d dg}tjdkrRtj |}n t j |}|j t ||_ xF|D]>}|j||dkr|j j|dd|d krr|j j|ddqrWdS)z1 Initialize configuration object < drnor!zapi.nixstats.comz/v2/server/pollz/var/log/nixstatsagent.loga) max_data_span max_data_age logging_levelthreadsttlintervalrr subprocessr%rr$api_pathlog_file log_file_modemax_cached_collectionsr executionr#rrjrg?NiX)r)loggingINFOr-r.rr/r* version_info configparserRawConfigParser ConfigParserr6 ini_filesr_config_section_createset)r^defaultssectionsrsectionrrrrXs<       zAgent._config_initcCs|jj|s|jj|dS)zc Create an addition section in the configuration object if it's not exists N)r has_section add_section)r^r|rrrrxs zAgent._config_section_createcCs|jjdd}|jjdd}|jjdd}|dkr4n |dkrBd}n|dkrPd}nd}|d krjtj|d n\ytj|||d d WnDtk r}z(tj|d tjd |tjdWYdd}~XnXtjd|dS)z, Initialize logging faculty rrgrmrnr)rdtruncateappend-)levelz,%(asctime)-15s %(levelname)s %(message)s)filenamefilemoderformatz IOError: %szDrop logging to stderrNzAgent logging_level %i)r)rd)rgetintrrq basicConfigIOErrorr)r^rrmrnrTrrrrZs&  zAgent._logging_initc Cs*tjd|jjdd}tjtjj|d}|tjkrDtjj d|i|_ x|D]}t |}|dkrfqP|j ||jj |drP|jj |drd|j |<qPtj|\}}}zJytj||||}Wn0tk rd}tjd ttjdYnXWd|r|jX|rd|j |<qPtjd |qPWdS) z& Discover the plugins r[rrz*.pyrrrkNzimport_plugin_exception:%szimport_plugin:%s)rqrrrglobr-r.rr*rFschedulerBrx getbooleanrGrHrJrIerrorstrexc_inforK) r^r filenamesrnamerQrRrSrUrrrr[s2     "  zAgent._plugins_initc Cs(tjtj|ftjtjdd}tjdtj|j |j j dd}t |}|j j |d}||p^d}|j xB|jdkr|d krtjd tj|tj||d8}|j qjW|jdkrtjd tj|j tj|j tj|j\}}|jd ks|rtjd tj||j||r tj|} nd} | S) z0 Execute /task/ in a subprocess T)stdoutstderruniversal_newlinesz %s:process:%irprjrirNrz %s:tick:%iz %s:kill:%iz %s:%s:%s:%s)rkPopenr* executablePIPErqdebug threading currentThreadpidrrrBpoll returncodetimesleeprr-killsignalSIGTERM communicatepickleloads) r^taskprocessrjrriZticksrrretrrr_subprocess_execution6s2         zAgent._subprocess_executioncCsbx@|jrtjdtjPtjdtj|jjy|jj}Wnt k rXPYnXtjdtj|t |}y|j j |d}Wnd}YnXt j }t|tr|j|}nry<|jj||jj |gi|j|j|d}|j|j }Wn4tk r&tjddttjd i}YnX|jj|||||d qW|jjtj|jjd S) zW Take queued execution requests, execute plugins and queue the results z %s:shutdownz%s:exec_queue:%iz %s:task:%srjr`)Z agent_cacheZplugin_exception exceptionr)tsrrrjrVN)shutdownrqrrrrexecuteqsize get_nowaitrrBrrrr=rrrYupdaterLrMrIrrr*rmetricsputcemeteryhirerelease)r^rrrjrrVrArrr _executionUsD    zAgent._executioncCstjdtj|jjdd}|jjdd}|jjdd}|jjdd}|jjdd}|jjdd }|jjdd }|jjdd }g} g} x\tj} |jrtjd tjPtj d tj|j j t | xX|j j r"y| j |j jWqtk r} ztjd| WYdd} ~ XqXqW| rtdd| D} tdd| D}tj}d}|| |kr|tj dd}d}n || |krtjdd}d}|rdd||fd}tj dtj| ddd|o|stjdd}nΐzy,tjd.krtjj|dd }ntj|dd }| rtjd!t | xj| r|jd"d#|tf| d$|d%|j}|j|j d&kr| d$=tj d'|j nt!d(|j q6Wtjd)|jd"d#|tft"j#t$tj| d*j%|d%|j}|j|j d&kr tj d'|j d}nt!d(|j Wntk r} zntjd+| |d$krt | |krh| d$=tjd,|tjd-| j t"j#t$tj| d*j%g} WYdd} ~ XnXWd|j&X|rg} |tj| }|d$krtj'|qWdS)/zA Take and collect data, send and clean if needed z%sr#r$rlrrfrerr%rjroz %s:shutdownz%s:data_queue:%i:collection:%izData queue error: %sNcss|]}|dVqdS)rNr).0rTrrr szAgent._data..css|]}|dVqdS)rNr)rrTrrrrsFz Max data spanTz Max data agezapplication/jsonz ApiKey %s:%s)z Content-typeZ Authorizationzcollection: %sr )rDrEz&Empty server or user, nowhere to send.r)timeoutzSending cached collections: %iPUTz %s?version=%sr)headerszSuccessful response: %szUnsuccessful response: %szAll cached collections sentrzFailed to submit collection: %szCReach max_cached_collections (%s): oldest cached collection droppedz,Cache current collection to resend next time)r)(rqrrrrrrrrrr#rr,rrrIrminmaxwarningrNrOr*rshttpclientZHTTPSConnectionhttplibZrequestrZ getresponser6status ValueErrorbz2compressrr5rKr)r^r$rlZmax_ageZmax_spanrr%rjroZcached_collectionsZ collectionZloop_tsrTZfirst_tsZlast_tsnowsendcleanr connectionresponsesleep_intervalrrr_datas$          " z Agent._datacCs tjdtj|jdjdS)z/ Initialize data worker thread r\)targetN)rqrrThreadrstart)r^rrrr\s zAgent._data_worker_initcCs<tjdkrtj}ntj}|jj|tjd|jdS)z, Dumps configuration object rz Config: %sN)r) r*rsioStringIOrr:rqrgetvalue)r^bufrrrr]s    zAgent._dump_configrcCs|jjdd}g}xztjtjj|dD]b}t|}|dkr>q(|j||dkrj|jj|dr|j |q(|dkr(|jj|ds(|j |q(W|S)z0 Return list with plugins names rrz*.pyrdisabled) rrrr-r.rrBrxrr)r^rrrrrPrrrrs  zAgent._get_pluginsc CsVtjd|jjx>y|jj}Wntk r8PYnXtjd||jqWdS)zb Join with dead workers Workaround for https://bugs.python.org/issue37788 z cemetery:%iz joining:%sN)rqrrrrrr)r^threadrrr_rips z Agent._ripc stjd|jjdd}tj|jjdd|_yx|jt j tj dtj xx|j j r|j j}|d}tj d||jd }|rt|jj|d|j|<t|tjr|j|d <|jj|qXWfd d |jjD}x|D]}tj d ||j|=|jj||jjd rzy&tj|jd}|jtj d|Wn0tk rv}ztjd|WYdd}~XnXqtjd|j jdd|jjddidqWdt j }|dkrt j |q6tjdt j dq6WWnt!k rtjt"j#dtjd|jd} x| rtj$} tjd| dd | D} | shtjdt"j%dd|_&tjdt'| x&| D]}tjd |||j(|qWq&WYn2tk r}ztj)d!|WYdd}~XnXdS)"z. Start all the worker threads zAgent main looprrjrprhz %i threadsrz metrics:%srcsg|]\}}|kr|qSrr)rwhatZwhen)rrr ?szAgent.run..z scheduling:%sF)rznew_execution_worker_thread:%szCan not start new thread: %sNZthreads_cappedZagent_internalZthreads_capping)rrrVg?rz'not enough time to start worker threadsg?z Shutting downTzRemaining threads: %scSs(g|] }|j rt|tj r|qSr)isDaemonr=r _MainThread)rrrrrrgs zBye!zWaiting for %i threads to exitzJoining with %s/%fzWorker error: %s)*rqrrgetfloatr Semaphorerrrrr activeCountrrrrintrr=types ModuleType__file__r#ritemsracquirerrrrIrrKeyboardInterruptr*r enumerateexitrr,rr) r^rjrrrArrrTrwait_forZ all_threadsr)rrrM&s|                     z Agent.runN)F)r)r@ __module__ __qualname__rrrr#rrr_rXrxrZr[rrrr\r]rrrMrrrrrs$ ("/j rcCsFttjdkr8tjdjdr8tjdddtjd<tjddkr^tdjdtjntjdd krtttjntjdd krtttjntjdd krtjd=tjt nrtjdd krtjd=tjt d dnJtjddkrtjt tjddn tdtjdtj dtjdn t j dS)Nrz--r helpr!Run without options to run agent.,Acceptable options (leading -- is optional):4 help, info, version, hello, insecure-hello, testrversionr<zinsecure-hellor)r;testzInvalid option:)file)rrr)r,r*r+ startswithr8rrrrr<rWrrrMrrrrmainws4     r__main__)r)r):Z __future__rrr*rsZ past.builtinsr ImportErrorrrtZ http.clientrqueuerrrrvrrrrGrNZ simplejsonrqr-rrr3rkrrrurllib urllib.parserrZurllib.requestr r Z urllib.errorr urllib2rr.dirnameabspathrr/rrwplatformrr<rBrWrrr@rrrrsz             ' (L"