pax_global_header00006660000000000000000000000064130372401500014505gustar00rootroot0000000000000052 comment=d7f6c40908fc1025390df9418d875beefa43bf7b nixstatsagent-1.1.4/000077500000000000000000000000001303724015000144045ustar00rootroot00000000000000nixstatsagent-1.1.4/.gitignore000066400000000000000000000001131303724015000163670ustar00rootroot00000000000000nixstats.ini *.pyc *~ build debian dist nixstatsagent.egg-info README.html nixstatsagent-1.1.4/.travis.yml000066400000000000000000000017211303724015000165160ustar00rootroot00000000000000language: python python: - '2.6' - '2.7' install: python setup.py install script: python setup.py sdist bdist_egg bdist_wheel deploy: provider: pypi user: btbroot password: secure: by+wKHR5IU8Rc4tyBSn2y8oH/Wz3nuw5NVSafpD+pLL+PjJ98AkYZFjVk7li79tu2OdhrpIR6+rHZSVcIP2EqyRMoxUgp45F158GYJmgpmY8Vi5NbNR0D/kP2qxMVTY3/J29bjns32RnjNU8PYumY0qg6kOZn/rh1LJHNasPcK2xrgAkTnplvHRS1IpFCoONN1hobg1a34CvvLic22KBPBAdOgAD8IxdVVbnrEpy7urxKld+ClJw3tvrqWgf17SNuYG+0vVr25fC/fcTESG17AiC/tw6i3IXrspJzR5dV+MGatWknokF4NaElXlDmf1wD1o2iY/8P4Xa9JTPLh56J5hvViAxBogafEuH566QgrufA6xXFSzS0+lkOkmYSK7kLWB9nEcDbVB8nMXt2RC7aLGzr0XZijeduhuADtO62mA9THgBEor56T/cSBi9fYYh9CC95glwTUN8UgDJ98+CcD8bVK1oFscMZgk/LqC6EMccOFJvi5zJshtd5y5/53XG/jqFUCvVPsW5XzA7r+9//l3RQeIcSfAQgkkpxb6bpoKctDwBGcjEMT0I8kwVtKJFjhw1Hfbbhg2QYyiRAhk0oaYGrHNxdYcf9te4BdtTqzoukHuJvx6NOjy7XyHh8C+SLhCbzkvMRHJVH92YhdsGsP1Lw6tFYzA/B75zwPhN/SM= on: tags: true distributions: sdist bdist_egg bdist_wheel repo: NIXStats/nixstatsagent nixstatsagent-1.1.4/LICENSE000066400000000000000000000030331303724015000154100ustar00rootroot00000000000000BSD Simplified License Copyright (c) 2016, NIXStats All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. nixstatsagent-1.1.4/MANIFEST.in000066400000000000000000000000571303724015000161440ustar00rootroot00000000000000include nixstats-example.ini LICENSE README.md nixstatsagent-1.1.4/README.md000066400000000000000000000040131303724015000156610ustar00rootroot00000000000000NixStats Agent ============== NixStats.com is a web service of monitoring and displaying statistics of your server performance. This software is an OS-agnostic agent compatible with Python 2.4, 2.5, 2.6 and 2.7. It's been optimized to have a small CPU consumption and comes with an extendable set of useful plugins. [![Build Status](https://travis-ci.org/NIXStats/nixstatsagent.svg?branch=master)](https://travis-ci.org/NIXStats/nixstatsagent) Installation ------------ Depending on your platform, many installation options are possible. We are listing them more or less in the order from the most specific (and preferred) to the most generic ones. ### Debian GNU/Linux The package is existing in Debian Sid and should appear soon in other releases and derivaives. If your suite is already supported, simply do: ``` apt-get install nixstatsagent ``` Until then, use our [packagecloud repository](https://packagecloud.io/btbroot/nixstats/install#bash) Current status: - [x] Ubuntu 16.04.1 LTS (packagecloud:ubuntu/xenial) - [x] Ubuntu 14.04.5 LTS (packagecloud:ubuntu/trusty) - [x] Ubuntu 12.04.5 LTS (packagecloud:ubuntu/precise) - [x] Debian 8 (packagecloud:debian/jessie) - [x] Debian 7 (packagecloud:debian/wheezy) - [ ] Debian 6 - [x] Debian 9 (sid, stretch, packagecloud:debian/stretch) - [x] Ubuntu 17.04 (zesty) ### Fedora - CentOS 7 - CentOS 6 - CentOS 5 - Fedora 25 (low priority) - Fedora 24 (low priority) - Fedora 23 (low priority) ### Windows - Windows 2016 (low priority) - Windows 2012 (low priority) ### Python 2.6 or 2.7 environment As the binary packages are published on [PyPI](https://pypi.python.org/pypi), provided that you've obtained [pip](https://pip.pypa.io/en/latest/installing/), simply do: ``` pip install nixstatsagent ``` ### Python 2.4 or 2.5 environment As the source package is published on [PyPI](https://pypi.python.org/pypi), provided that you've obtained [setuptools](https://pypi.python.org/pypi/setuptools#installation-instructions), simply do: ``` easy_install nixstatsagent ``` nixstatsagent-1.1.4/nixstats-example.ini000066400000000000000000000025261303724015000204200ustar00rootroot00000000000000# [DEFAULT] ; Values here *override* the hardcoded defaults (listed) # max_data_span = 60 ; Collected data span threshold for sending, sec # max_data_age = 600 ; Collected data age threshold for sending, sec # logging_level = 30 ; (logging.WARNING) Logging level, defined in logging # threads = 100 ; Maximun amount of threads # interval = 60 ; Interval betveen iterations, sec # plugins = plugins ; Path to plugins # enabled = no ; Toggle plugin # subprocess = no ; Run plugin as a subprocess # ttl = 60 ; Time to live (for a subprocess), sec # user = '' ; API user name # server = '' ; API server name # # ###################### Default sections # [agent] ; Main thread # # [execution] ; Plugin execution threads # ttl = 15 ; Example: plugins will be killed after 15 sec # # [data] ; Collected data sening thread # interval = 600 ; Example: the data collected will be checked every minute for thresholds # # ##################### Plugin sections examples # [cpu] ; Example: Shortname for 'plugins/cpu.py' plugin # enabled = yes ; Example: cpu plugin enabled # ttl = 5 ; Example: cpu plugin time to live # # [sleeper] ; Example: Shortname for 'plugins/sleeper.py' plugin # enabled = yes ; Example: sleeper plugin disabled nixstatsagent-1.1.4/nixstatsagent/000077500000000000000000000000001303724015000173005ustar00rootroot00000000000000nixstatsagent-1.1.4/nixstatsagent/__init__.py000066400000000000000000000000001303724015000213770ustar00rootroot00000000000000nixstatsagent-1.1.4/nixstatsagent/nixstatsagent.py000077500000000000000000000311031303724015000225470ustar00rootroot00000000000000#!/usr/bin/env python # by Al Nikolov import bz2 import ConfigParser import glob import httplib import imp try: import json as serialize except ImportError: import simplejson as serialize import logging import os import pickle import Queue import signal import socket import StringIO import subprocess import sys import threading import time import types import urllib import urllib2 ini_files = ( os.path.join('/etc', 'nixstats.ini'), os.path.join('/etc', 'nixstats-token.ini'), os.path.abspath('nixstats.ini'), os.path.abspath('nixstats-token.ini'), ) def hello(proto='https'): user_id = sys.argv[1] token_filename = sys.argv[2] if len(sys.argv) > 2 else 'nixstats-token.ini' try: hostname = os.uname()[1] except AttributeError: hostname = socket.getfqdn() server_id = urllib2.urlopen( proto + '://api.nixstats.com/hello.php', data=urllib.urlencode( { 'user': user_id, 'hostname': hostname } ) ).read() print('Got server_id: {}'.format(server_id)) open(token_filename, 'w').\ write('[DEFAULT]\nuser={}\nserver={}\n'.format(user_id, server_id)) def run_agent(): Agent().run() def _plugin_name(plugin): if isinstance(plugin, basestring): basename = os.path.basename(plugin) return os.path.splitext(basename)[0] else: return plugin.__name__ class Agent: execute = Queue.Queue() metrics = Queue.Queue() data = Queue.Queue() shutdown = False def __init__(self): """ Initialize internal strictures """ self._config_init() self._logging_init() self._plugins_init() self._data_worker_init() self._dump_config() def _config_init(self): """ Initialize configuration object """ defaults = { 'max_data_span': 60, 'max_data_age': 60 * 10, 'logging_level': logging.WARNING, 'threads': 100, 'ttl': 60, 'interval': 60, 'plugins': 'plugins', 'enabled': 'no', 'subprocess': 'no', 'user': '', 'server': '', 'api_host': 'api.nixstats.com', 'api_path': '/v2/server/poll', } sections = [ 'agent', 'execution', 'data', ] config = ConfigParser.RawConfigParser(defaults) config.read(ini_files) self.config = config for section in sections: self._config_section_create(section) def _config_section_create(self, section): """ Create an addition section in the configuration object """ if not self.config.has_section(section): self.config.add_section(section) def _logging_init(self): """ Initialize logging faculty """ level = self.config.getint('agent', 'logging_level') logging.basicConfig(level=level) logging.info('Agent logging_level %i', level) def _plugins_init(self): """ Discover the plugins """ logging.info('_plugins_init') path = self.config.get('agent', 'plugins') filenames = glob.glob(os.path.join(path, '*.py')) sys.path.insert(0, path) self.schedule = {} for filename in filenames: name = _plugin_name(filename) if name == 'plugins': continue self._config_section_create(name) if self.config.getboolean(name, 'enabled'): if self.config.getboolean(name, 'subprocess'): self.schedule[filename] = 0 else: fp, pathname, description = \ imp.find_module(name) module = None try: module = imp.load_module( name, fp, pathname, description) finally: # Since we may exit via an exception, close fp explicitly. if fp: fp.close() if module: self.schedule[module] = 0 else: logging.error('import_plugin:%s:%s', name, sys.exc_type) def _subprocess_execution(self, task): """ Execute /task/ in a subprocess """ process = subprocess.Popen((sys.executable, task, ini_file), stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) logging.info('%s:process:%i', threading.currentThread(), process.pid) interval = self.config.getint('execution', 'interval') name = _plugin_name(task) ttl = self.config.getint(name, 'ttl') ticks = ttl / interval or 1 process.poll() while process.returncode is None and ticks > 0: logging.info('%s:tick:%i', threading.currentThread(), ticks) time.sleep(interval) ticks -= 1 process.poll() if process.returncode is None: logging.error('%s:kill:%i', threading.currentThread(), process.pid) os.kill(process.pid, signal.SIGTERM) stdout, stderr = process.communicate() if process.returncode != 0 or stderr: logging.error('%s:%s:%s:%s', threading.currentThread(), task, process.returncode, stderr) if stdout: ret = pickle.loads(stdout) else: ret = None return ret def _execution(self): """ Take queued execution requests, execute plugins and queue the results """ while True: if self.shutdown: logging.info('%s:shutdown', threading.currentThread()) break logging.info('%s:exec_queue:%i', threading.currentThread(), self.execute.qsize()) try: task = self.execute.get_nowait() except Queue.Empty: break logging.info('%s:task:%s', threading.currentThread(), task) name = _plugin_name(task) ts = time.time() if isinstance(task, basestring): payload = self._subprocess_execution(task) else: try: payload = task.Plugin().run(self.config) except: logging.exception('plugin_exception') payload = { 'exception': str(sys.exc_info()[0]) } self.metrics.put({ 'ts': ts, 'task': task, 'name': name, 'payload': payload, }) self.hire.release() def _data(self): """ Take and collect data, send and clean if needed """ logging.info('%s', threading.currentThread()) collection = [] while True: if self.shutdown: logging.info('%s:shutdown', threading.currentThread()) break logging.info('%s:data_queue:%i:collection:%i', threading.currentThread(), self.data.qsize(), len(collection)) while self.data.qsize(): collection.append(self.data.get_nowait()) if collection: first_ts = min((e['ts'] for e in collection)) last_ts = max((e['ts'] for e in collection)) now = time.time() max_age = self.config.getint('agent', 'max_data_age') max_span = self.config.getint('agent', 'max_data_span') send = False server = self.config.get('agent', 'server') user = self.config.get('agent', 'user') if last_ts - first_ts > max_span: logging.info('Max data span') send = True elif now - first_ts > max_age: logging.info('Max data age') send = True if send: headers = { "Content-type": "application/json", "Authorization": 'ApiKey %s:%s' % ( self.config.get('agent', 'user'), self.config.get('agent', 'server')), } clean = False logging.info('collection:%s', collection) if not server and not user: logging.info('Empty server/user, but need to send: %s', serialize.dumps(collection)) clean = True else: connection = httplib.HTTPSConnection(self.config.get('data', 'api_host')) connection.request('PUT', self.config.get('data', 'api_path'), bz2.compress(str(serialize.dumps(collection)) + "\n"), headers=headers) response = connection.getresponse() logging.info('%s', response) connection.close() if response.status == 200: clean = True if clean: collection = [] time.sleep(self.config.getint('data', 'interval')) def _data_worker_init(self): """ Initialize data worker thread """ logging.info('_data_worker_init') threading.Thread(target=self._data).start() def _dump_config(self): """ Dumps configuration object """ buf = StringIO.StringIO() self.config.write(buf) logging.info('Config: %s', buf.getvalue()) def run(self): """ Start all the worker threads """ logging.info('Agent main loop') interval = self.config.getint('agent', 'interval') self.hire = threading.Semaphore( self.config.getint('execution', 'threads')) try: while True: now = time.time() logging.info('%i threads', threading.activeCount()) while self.metrics.qsize(): metrics = self.metrics.get_nowait() name = metrics['name'] logging.info('metrics:%s', name) plugin = metrics.get('task') if plugin: self.schedule[plugin] = \ now + self.config.getint(name, 'interval') if isinstance(plugin, types.ModuleType): metrics['task'] = plugin.__file__ self.data.put(metrics) execute = [what for what, when in self.schedule.items() if when <= now ] for name in execute: logging.info('scheduling:%s', name) del self.schedule[name] self.execute.put(name) if self.hire.acquire(False): thread = threading.Thread(target=self._execution) thread.start() logging.info('new_execution_worker_thread:%s', thread) else: logging.warning('threads_capped') self.metrics.put({ 'ts': now, 'name': 'agent_internal', 'payload': { 'threads_capping': self.config.getint('execution', 'threads')} }) time.sleep(interval) except KeyboardInterrupt: logging.warning(sys.exc_info()[0]) while True: wait_for = [thread for thread in threading.enumerate() if not thread.isDaemon() and not isinstance(thread, threading._MainThread)] logging.info('Shutdown, waiting for %i threads to exit', len(wait_for)) logging.info('Remaining threads: %s', threading.enumerate()) if len(wait_for) == 0: sys.exit(0) self.shutdown = True time.sleep(interval) if __name__ == '__main__': if len(sys.argv) > 1: if sys.argv[1] == 'hello': del sys.argv[1] hello() elif sys.argv[1] == 'insecure-hello': del sys.argv[1] hello(proto='http') else: run_agent() nixstatsagent-1.1.4/nixstatsagent/plugins/000077500000000000000000000000001303724015000207615ustar00rootroot00000000000000nixstatsagent-1.1.4/nixstatsagent/plugins/__init__.py000066400000000000000000000000001303724015000230600ustar00rootroot00000000000000nixstatsagent-1.1.4/nixstatsagent/plugins/cpu.py000077500000000000000000000011001303724015000221150ustar00rootroot00000000000000#!/usr/bin/env python import psutil import plugins class Plugin(plugins.BasePlugin): def run(self, *unused): name = 'cpu' results = {} data = psutil.cpu_times_percent(interval=1, percpu=True) cpu_number = -1 for cpu in data: core = {} cpu_number = cpu_number+1 results[cpu_number] = {} for key in cpu._fields: core[key] = getattr(cpu, key) results[cpu_number] = core return results if __name__ == '__main__': Plugin().execute() nixstatsagent-1.1.4/nixstatsagent/plugins/diskusage.py000077500000000000000000000023671303724015000233250ustar00rootroot00000000000000#!/usr/bin/env python import os import sys import psutil import plugins class Plugin(plugins.BasePlugin): def run(self, *unused): disk = {} if sys.platform == "linux" or sys.platform == "linux2" or sys.platform == "darwin": disk['df'] = [s.split() for s in os.popen("df -Pl").read().splitlines()] disk['di'] = [s.split() for s in os.popen("df -iPl").read().splitlines()] elif sys.platform == "win32": disk = {} disk['df-windows'] = [] for part in psutil.disk_partitions(all=False): if os.name == 'nt': if 'cdrom' in part.opts or part.fstype == '': # skip cd-rom drives with no disk in it; they may raise # ENOENT, pop-up a Windows GUI error for a non-ready # partition or just hang. continue usage = psutil.disk_usage(part.mountpoint) diskdata = {} diskdata['info'] = part for key in usage._fields: diskdata[key] = getattr(usage, key) disk['df-windows'].append(diskdata) return disk if __name__ == '__main__': Plugin().execute() nixstatsagent-1.1.4/nixstatsagent/plugins/iostat.py000077500000000000000000000207101303724015000226410ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf-8 # # collectd-iostat-python # ====================== # # Collectd-iostat-python is an iostat plugin for collectd that allows you to # graph Linux iostat metrics in Graphite or other output formats that are # supported by collectd. # # https://github.com/powdahound/redis-collectd-plugin # - was used as template # https://github.com/keirans/collectd-iostat/ # - was used as inspiration and contains some code from # https://bitbucket.org/jakamkon/python-iostat # - by Kuba Kończyk # import os import signal import subprocess import sys import psutil import plugins __version__ = '0.0.3' __author__ = 'denis.zhdanov@gmail.com' class IOStatError(Exception): pass class CmdError(IOStatError): pass class ParseError(IOStatError): pass class IOStat(object): def __init__(self, path='iostat', interval=2, count=2, disks=[]): self.path = path self.interval = interval self.count = count self.disks = disks def parse_diskstats(self, input): """ Parse iostat -d and -dx output.If there are more than one series of statistics, get the last one. By default parse statistics for all avaliable block devices. @type input: C{string} @param input: iostat output @type disks: list of C{string}s @param input: lists of block devices that statistics are taken for. @return: C{dictionary} contains per block device statistics. Statistics are in form of C{dictonary}. Main statistics: tps Blk_read/s Blk_wrtn/s Blk_read Blk_wrtn Extended staistics (available with post 2.5 kernels): rrqm/s wrqm/s r/s w/s rsec/s wsec/s rkB/s wkB/s avgrq-sz \ avgqu-sz await svctm %util See I{man iostat} for more details. """ dstats = {} dsi = input.rfind('Device:') if dsi == -1: raise ParseError('Unknown input format: %r' % input) ds = input[dsi:].splitlines() hdr = ds.pop(0).split()[1:] for d in ds: if d: d = d.split() dev = d.pop(0) if (dev in self.disks) or not self.disks: dstats[dev] = dict([(k, float(v)) for k, v in zip(hdr, d)]) return dstats def sum_dstats(self, stats, smetrics): """ Compute the summary statistics for chosen metrics. """ avg = {} for disk, metrics in stats.iteritems(): for mname, metric in metrics.iteritems(): if mname not in smetrics: continue if mname in avg: avg[mname] += metric else: avg[mname] = metric return avg def _run(self, options=None): """ Run iostat command. """ close_fds = 'posix' in sys.builtin_module_names args = '%s %s %s %s %s' % ( self.path, ''.join(options), self.interval, self.count, ' '.join(self.disks)) return subprocess.Popen( args, bufsize=1, shell=True, stdout=subprocess.PIPE, close_fds=close_fds) @staticmethod def _get_childs_data(child): """ Return child's data when avaliable. """ (stdout, stderr) = child.communicate() ecode = child.poll() if ecode != 0: raise CmdError('Command %r returned %d' % (child.cmd, ecode)) return stdout def get_diskstats(self): """ Get all avaliable disks statistics that we can get. """ dstats = self._run(options=['-kNd']) extdstats = self._run(options=['-kNdx']) dsd = self._get_childs_data(dstats) edd = self._get_childs_data(extdstats) ds = self.parse_diskstats(dsd) eds = self.parse_diskstats(edd) for dk, dv in ds.iteritems(): if dk in eds: ds[dk].update(eds[dk]) return ds class IOMon(object): def __init__(self): self.plugin_name = 'collectd-iostat-python' self.iostat_path = '/usr/bin/iostat' self.iostat_interval = 2 self.iostat_count = 2 self.iostat_disks = [] self.iostat_nice_names = False self.iostat_disks_regex = '' self.verbose_logging = False self.names = { 'tps': {'t': 'transfers_per_second'}, 'Blk_read/s': {'t': 'blocks_per_second', 'ti': 'read'}, 'kB_read/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 10e3}, 'MB_read/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 10e6}, 'Blk_wrtn/s': {'t': 'blocks_per_second', 'ti': 'write'}, 'kB_wrtn/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 10e3}, 'MB_wrtn/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 10e6}, 'Blk_read': {'t': 'blocks', 'ti': 'read'}, 'kB_read': {'t': 'bytes', 'ti': 'read', 'm': 10e3}, 'MB_read': {'t': 'bytes', 'ti': 'read', 'm': 10e6}, 'Blk_wrtn': {'t': 'blocks', 'ti': 'write'}, 'kB_wrtn': {'t': 'bytes', 'ti': 'write', 'm': 10e3}, 'MB_wrtn': {'t': 'bytes', 'ti': 'write', 'm': 10e6}, 'rrqm/s': {'t': 'requests_merged_per_second', 'ti': 'read'}, 'wrqm/s': {'t': 'requests_merged_per_second', 'ti': 'write'}, 'r/s': {'t': 'per_second', 'ti': 'read'}, 'w/s': {'t': 'per_second', 'ti': 'write'}, 'rsec/s': {'t': 'sectors_per_second', 'ti': 'read'}, 'rkB/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 10e3}, 'rMB/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 10e6}, 'wsec/s': {'t': 'sectors_per_second', 'ti': 'write'}, 'wkB/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 10e3}, 'wMB/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 10e6}, 'avgrq-sz': {'t': 'avg_request_size'}, 'avgqu-sz': {'t': 'avg_request_queue'}, 'await': {'t': 'avg_wait_time'}, 'r_await': {'t': 'avg_wait_time', 'ti': 'read'}, 'w_await': {'t': 'avg_wait_time', 'ti': 'write'}, 'svctm': {'t': 'avg_service_time'}, '%util': {'t': 'percent', 'ti': 'util'} } def restore_sigchld(): """ Restore SIGCHLD handler for python <= v2.6 It will BREAK exec plugin!!! See https://github.com/deniszh/collectd-iostat-python/issues/2 for details """ if sys.version_info[0] == 2 and sys.version_info[1] <= 6: signal.signal(signal.SIGCHLD, signal.SIG_DFL) def diskstats_parse(dev=None): file_path = '/proc/diskstats' result = {} # ref: http://lxr.osuosl.org/source/Documentation/iostats.txt columns_disk = ['m', 'mm', 'dev', 'reads', 'rd_mrg', 'rd_sectors', 'ms_reading', 'writes', 'wr_mrg', 'wr_sectors', 'ms_writing', 'cur_ios', 'ms_doing_io', 'ms_weighted'] columns_partition = ['m', 'mm', 'dev', 'reads', 'rd_sectors', 'writes', 'wr_sectors'] lines = open(file_path, 'r').readlines() for line in lines: if line == '': continue split = line.split() if len(split) == len(columns_disk): columns = columns_disk elif len(split) == len(columns_partition): columns = columns_partition else: # No match continue data = dict(zip(columns, split)) if "loop" in data['dev']: continue if dev != None and dev != data['dev']: continue for key in data: if key != 'dev': data[key] = int(data[key]) result[data['dev']] = data return result class Plugin(plugins.BasePlugin): def run(self, *unused): if(os.path.isfile("/proc/diskstats")): return diskstats_parse() elif(os.path.isfile("/usr/bin/iostat")): iostat = IOStat() ds = iostat.get_diskstats() if ds: return ds else: results = {} diskdata = psutil.disk_io_counters(perdisk=True) for device,values in diskdata.items(): device_stats = {} for key_value in values._fields: device_stats[key_value] = getattr(values, key_value) results[device] = device_stats return results if __name__ == '__main__': Plugin().execute() nixstatsagent-1.1.4/nixstatsagent/plugins/loadavg.py000077500000000000000000000003061303724015000227520ustar00rootroot00000000000000#!/usr/bin/env python import os import plugins class Plugin(plugins.BasePlugin): def run(self, *unused): return os.getloadavg() if __name__ == '__main__': Plugin().execute() nixstatsagent-1.1.4/nixstatsagent/plugins/memory.py000077500000000000000000000005131303724015000226450ustar00rootroot00000000000000#!/usr/bin/env python import psutil import plugins class Plugin(plugins.BasePlugin): def run(self, *unused): memory = {} mem = psutil.virtual_memory() for name in mem._fields: memory[name] = getattr(mem, name) return memory if __name__ == '__main__': Plugin().execute() nixstatsagent-1.1.4/nixstatsagent/plugins/network.py000077500000000000000000000003411303724015000230250ustar00rootroot00000000000000#!/usr/bin/env python import psutil import plugins class Plugin(plugins.BasePlugin): def run(self, *unused): return psutil.net_io_counters(pernic=True) if __name__ == '__main__': Plugin().execute() nixstatsagent-1.1.4/nixstatsagent/plugins/ping.py000077500000000000000000000052221303724015000222740ustar00rootroot00000000000000#!/usr/bin/env python # coding: utf-8 import re from subprocess import Popen, PIPE import sys import plugins def _get_match_groups(ping_output, regex): match = regex.search(ping_output) if not match: return False else: return match.groups() def system_command(Command, newlines=True): Output = "" Error = "" try: #Output = subprocess.check_output(Command,stderr = subprocess.STDOUT,shell='True') proc = Popen(Command.split(), stdout=PIPE) Output = proc.communicate()[0] except: pass if Output: if newlines == True: Stdout = Output.split("\\n") else: Stdout = Output else: Stdout = [] if Error: Stderr = Error.split("\n") else: Stderr = [] return (Stdout,Stderr) def collect_ping(hostname): if sys.platform == "linux" or sys.platform == "linux2": response = str(system_command("ping -W 5 -c 1 " + hostname, False)[0]) matcher = re.compile(r'(\d+.\d+)/(\d+.\d+)/(\d+.\d+)/(\d+.\d+)') minping, avgping, maxping, jitter = _get_match_groups(response, matcher) response = avgping elif sys.platform == "darwin": # print system_command("ping -W 5 -c 2 " + hostname, False) response = str(system_command("ping -c 1 " + hostname, False)[0]) # matcher = re.compile(r'min/avg/max/stddev = (\d+)/(\d+)/(\d+)/(\d+) ms') # min, avg, max, stddev = _get_match_groups(response, matcher) matcher = re.compile(r'(\d+.\d+)/(\d+.\d+)/(\d+.\d+)/(\d+.\d+)') matched = _get_match_groups(response, matcher) if matched == False: response = 0 else: minping, avgping, maxping, jitter = matched response = avgping elif sys.platform == "win32": response = 0 try: ping = Popen(["ping", "-n", "1 ", hostname], stdout=PIPE, stderr=PIPE) out, error = ping.communicate() if out: try: response = int(re.findall(r"Average = (\d+)", out)[0]) except: pass else: response = 0 except CalledProcessError: pass else: response = system_command("ping -W -c 1 " + hostname) return {'avgping': response, 'host': hostname} class Plugin(plugins.BasePlugin): def run(self, config): data = {} my_hosts = config.get('ping', 'hosts').split(',') data['ping'] = [] for host in my_hosts: data['ping'].append(collect_ping(host)) return data['ping'] if __name__ == '__main__': Plugin().execute() nixstatsagent-1.1.4/nixstatsagent/plugins/plugins.py000066400000000000000000000010521303724015000230120ustar00rootroot00000000000000import ConfigParser import pickle import sys class BasePlugin: """ Abstract class for plugins """ name = '' def run(self, config=None): """ Virtual method for running the plugin """ pass def execute(self): """ Execution wrapper for the plugin argv[1]: ini_file """ config = None if len(sys.argv) > 1: config = ConfigParser.RawConfigParser() config.read(sys.argv[1]) pickle.dump(self.run(config), sys.stdout) nixstatsagent-1.1.4/nixstatsagent/plugins/process.py000066400000000000000000000070501303724015000230130ustar00rootroot00000000000000#!/usr/bin/env python import netifaces import os import platform from subprocess import Popen, PIPE import sys try: import pwd except ImportError: pass import psutil import plugins def procStats(pid): process = {} process['pid'] = int(pid) if os.path.exists('/proc/%s/cmdline' % pid): with open(os.path.join('/proc/', pid, 'cmdline'), 'r') as file: process['cmdline'] = file.readline().replace('\x00', ' ') if process['cmdline'].strip() == '': return False else: if os.path.exists('/proc/%s/stat' % pid): pidfile = open(os.path.join('/proc/', pid, 'stat')) if pidfile: proctimes = pidfile.readline() process['name'] = proctimes.split(' ')[1].strip(')').strip('(') # count total process used time process['ctime'] = float(int(proctimes.split(' ')[13]) + int(proctimes.split(' ')[14])) if os.path.exists('/proc/%s/io' % pid): process['io'] = {} f = open(os.path.join('/proc/', pid, 'io')) if f: for line in f: process['io'][line.strip().split(': ')[0]] = line.strip().split(': ')[1] if os.path.exists('/proc/%s/status' % pid): f = open(os.path.join('/proc/', pid, 'status')) if f: for line in f: if line.split(':')[0] == 'Uid': process['username'] = pwd.getpwuid(int(line.split(':')[1].split('\t')[1])).pw_name if line.split(':')[0] == 'PPid': process['ppid'] = int(line.split(':')[1].split('\t')[1]) if line.split(':')[0] == 'VmSize': process['vmsize'] = line.split(':')[1].split('kB')[0].strip() if line.split(':')[0] == 'State': process['cmdline'] += line.split(':')[1].strip() if line.split(':')[0] == 'VmRSS': process['vmrss'] = line.split(':')[1].split('kB')[0].strip() break return process def getPids(): return [int(x) for x in os.listdir(b'/proc') if x.isdigit()] class Plugin(plugins.BasePlugin): def run(self, *unused): process = [] if sys.platform == "linux" or sys.platform == "linux2": processlist = [] for pid in getPids(): proc = procStats(str(pid)) if proc != False: process.append(proc) elif sys.platform == "darwin": for proc in psutil.process_iter(): try: pinfo = proc.as_dict(attrs=['pid', 'name', 'ppid', 'exe', 'cmdline', 'username', 'cpu_percent','memory_percent']) except psutil.NoSuchProcess: pass else: if ''.join(pinfo['cmdline']) != '': process.append(pinfo) else: for proc in psutil.process_iter(): try: pinfo = proc.as_dict(attrs=['pid', 'name', 'ppid', 'exe', 'cmdline', 'username', 'cpu_percent','memory_percent']) except psutil.NoSuchProcess: pass else: process.append(pinfo) return process if __name__ == '__main__': Plugin().execute() nixstatsagent-1.1.4/nixstatsagent/plugins/sleeper.py000077500000000000000000000003131303724015000227720ustar00rootroot00000000000000#!/usr/bin/env python import time import plugins class Plugin(plugins.BasePlugin): def run(self, *unused): time.sleep(60 * 60 * 24) if __name__ == '__main__': Plugin().run() nixstatsagent-1.1.4/nixstatsagent/plugins/swap.py000077500000000000000000000005011303724015000223040ustar00rootroot00000000000000#!/usr/bin/env python import psutil import plugins class Plugin(plugins.BasePlugin): def run(self, *unused): swap = {} mem = psutil.swap_memory() for name in mem._fields: swap[name] = getattr(mem, name) return swap if __name__ == '__main__': Plugin().execute() nixstatsagent-1.1.4/nixstatsagent/plugins/system.py000077500000000000000000000063321303724015000226660ustar00rootroot00000000000000#!/usr/bin/env python import netifaces import os import platform from subprocess import Popen, PIPE import sys import time import psutil import plugins def systemCommand(Command, newlines=True): Output = "" Error = "" try: #Output = subprocess.check_output(Command,stderr = subprocess.STDOUT,shell='True') proc = Popen(Command.split(), stdout=PIPE) Output = proc.communicate()[0] except: pass if Output: if newlines == True: Stdout = Output.split("\n") else: Stdout = Output else: Stdout = [] if Error: Stderr = Error.split("\n") else: Stderr = [] return (Stdout,Stderr) def ip_addresses(): ip_list = {} ip_list['v4'] = {} ip_list['v6'] = {} for interface in netifaces.interfaces(): link = netifaces.ifaddresses(interface) if netifaces.AF_INET in link: if interface not in ip_list['v4']: ip_list['v4'][interface] = [] ip_list['v4'][interface].append(link[netifaces.AF_INET]) if netifaces.AF_INET6 in link: if interface not in ip_list['v6']: ip_list['v6'][interface] = [] ip_list['v6'][interface].append(link[netifaces.AF_INET6]) return ip_list class Plugin(plugins.BasePlugin): def run(self, *unused): systeminfo = {} cpu = {} if(os.path.isfile("/proc/cpuinfo")): f = open('/proc/cpuinfo') if f: for line in f: # Ignore the blank line separating the information between # details about two processing units if line.strip(): if "model name" == line.rstrip('\n').split(':')[0].strip(): cpu['brand'] = line.rstrip('\n').split(':')[1].strip() if "processor" == line.rstrip('\n').split(':')[0].strip(): cpu['count'] = line.rstrip('\n').split(':')[1].strip() else: cpu['brand'] = "Unknown CPU" cpu['count'] = 0 mem = psutil.virtual_memory() if sys.platform == "linux" or sys.platform == "linux2": systeminfo['os'] = str(' '.join(platform.linux_distribution())) elif sys.platform == "darwin": systeminfo['os'] = "Mac OS %s" % platform.mac_ver()[0] cpu['brand'] = str(systemCommand('sysctl machdep.cpu.brand_string', False)[0]).split(': ')[1] cpu['count'] = systemCommand('sysctl hw.ncpu') elif sys.platform == "freebsd10": systeminfo['os'] = "FreeBSD %s" % platform.release() cpu['brand'] = str(systemCommand('sysctl hw.model', False)[0]).split(': ')[1] cpu['count'] = systemCommand('sysctl hw.ncpu') elif sys.platform == "win32": systeminfo['os'] = "{} {}".format(platform.uname()[0],platform.uname()[2]) systeminfo['cpu'] = cpu['brand'] systeminfo['cores'] = cpu['count'] systeminfo['memory'] = mem.total systeminfo['uptime'] = int(time.time()-psutil.boot_time()) systeminfo['ip_addresses'] = ip_addresses() return systeminfo if __name__ == '__main__': Plugin().execute() nixstatsagent-1.1.4/setup.py000077500000000000000000000033221303724015000161210ustar00rootroot00000000000000#!/usr/bin/env python # by Al Nikolov import glob import os import sys import setuptools here = os.path.abspath(os.path.dirname(__file__)) readme = open(os.path.join(here, 'README.md')).read() if sys.version.startswith('2.4'): install_requires=['psutil==2.1.3', 'netifaces==0.8', 'simplejson==2.1.0'] else: install_requires=['psutil', 'netifaces'] setuptools.setup( name='nixstatsagent', version='1.1.4', description='NixStats agent', long_description=readme, url='https://github.com/NIXStats/nixstatsagent', author='NIXStats', author_email='vincent@nixstats.com', maintainer='Al Nikolov', maintainer_email='root@toor.fi.eu.org', license='BSD-3-Clause', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: No Input/Output (Daemon)', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: BSD License', 'Natural Language :: English', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python :: 2.4', 'Programming Language :: Python :: 2.5', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Topic :: System :: Monitoring', ], keywords='nixstats system monitoring agent', install_requires=install_requires, packages=setuptools.find_packages(), entry_points={ 'console_scripts': [ 'nixstatsagent=nixstatsagent.nixstatsagent:run_agent', 'nixstatshello=nixstatsagent.nixstatsagent:hello', ], }, data_files=[('share/doc/nixstatsagent', [ 'nixstats-example.ini', 'LICENSE', 'README.md', ])], )