'''':; exec "$(command -v python || command -v python3 || command -v python2 || echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" # '''

import os

import sys

import time

import threading

MODULE_EXTENSION = ".chart.py"

BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),

'priority': 90000,

'retries': 10}

MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',

os.path.dirname(__file__)) + "/../python.d") + "/"

CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")

if CONFIG_DIR[-1] != "/":

CONFIG_DIR += "/"

sys.path.append(MODULES_DIR + "python_modules")

PROGRAM = os.path.basename(__file__).replace(".plugin", "")

DEBUG_FLAG = False

OVERRIDE_UPDATE_EVERY = False

import msg

try:

assert sys.version_info >= (3, 1)

import importlib.machinery

PY_VERSION = 3

# change this hack below if we want PY_VERSION to be used in modules

# import builtins

# builtins.PY_VERSION = 3

msg.info('Using python v3')

except (AssertionError, ImportError):

try:

import imp

# change this hack below if we want PY_VERSION to be used in modules

# import __builtin__

# __builtin__.PY_VERSION = 2

PY_VERSION = 2

msg.info('Using python v2')

except ImportError:

msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')

try:

if PY_VERSION == 3:

import pyyaml3 as yaml

else:

import pyyaml2 as yaml

except ImportError:

msg.fatal('Cannot find yaml library')

class PythonCharts(object):

"""

def __init__(self,

modules=None,

modules_path='../python.d/',

modules_configs='../conf.d/',

modules_disabled=None):

"""

if modules is None:

modules = []

if modules_disabled is None:

modules_disabled = []

self.first_run = True

# set configuration directory

self.configs = modules_configs

# load modules

loaded_modules = self._load_modules(modules_path, modules, modules_disabled)

# load configuration files

configured_modules = self._load_configs(loaded_modules)

# good economy and prosperity:

self.jobs = self._create_jobs(configured_modules) # type: list

# enable timetable override like `python.d.plugin mysql debug 1`

if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:

for job in self.jobs:

job.create_timetable(BASE_CONFIG['update_every'])

@staticmethod

def _import_module(path, name=None):

"""

if name is None:

name = path.split('/')[-1]

if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:

return None

name = name[:-len(MODULE_EXTENSION)]

try:

if PY_VERSION == 3:

return importlib.machinery.SourceFileLoader(name, path).load_module()

else:

return imp.load_source(name, path)

except Exception as e:

msg.error("Problem loading", name, str(e))

return None

def _load_modules(self, path, modules, disabled):

"""

# check if plugin directory exists

if not os.path.isdir(path):

msg.fatal("cannot find charts directory ", path)

# load modules

loaded = []

if len(modules) > 0:

for m in modules:

if m in disabled:

continue

mod = self._import_module(path + m + MODULE_EXTENSION)

if mod is not None:

loaded.append(mod)

else: # exit if plugin is not found

msg.fatal('no modules found.')

else:

# scan directory specified in path and load all modules from there

names = os.listdir(path)

for mod in names:

if mod.replace(MODULE_EXTENSION, "") in disabled:

msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, ""))

continue

m = self._import_module(path + mod)

if m is not None:

msg.debug(mod + ": loading module '" + path + mod + "'")

loaded.append(m)

return loaded

def _load_configs(self, modules):

"""

for mod in modules:

configfile = self.configs + mod.__name__ + ".conf"

if os.path.isfile(configfile):

msg.debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")

try:

if not hasattr(mod, 'config'):

mod.config = {}

setattr(mod,

'config',

self._parse_config(mod, read_config(configfile)))

except Exception as e:

msg.error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))

else:

msg.error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")

# set config if not found

if not hasattr(mod, 'config'):

msg.debug(mod.__name__ + ": setting configuration for only one job")

mod.config = {None: {}}

for var in BASE_CONFIG:

try:

mod.config[None][var] = getattr(mod, var)

except AttributeError:

mod.config[None][var] = BASE_CONFIG[var]

return modules

@staticmethod

def _parse_config(module, config):

"""

if config is None:

config = {}

# get default values

defaults = {}

msg.debug(module.__name__ + ": reading configuration")

for key in BASE_CONFIG:

try:

# get defaults from module config

defaults[key] = int(config.pop(key))

except (KeyError, ValueError):

try:

# get defaults from module source code

defaults[key] = getattr(module, key)

except (KeyError, ValueError, AttributeError):

# if above failed, get defaults from global dict

defaults[key] = BASE_CONFIG[key]

# check if there are dict in config dict

many_jobs = False

for name in config:

if type(config[name]) is dict:

many_jobs = True

break

# assign variables needed by supervisor to every job configuration

if many_jobs:

for name in config:

for key in defaults:

if key not in config[name]:

config[name][key] = defaults[key]

# if only one job is needed, values doesn't have to be in dict (in YAML)

else:

config = {None: config.copy()}

config[None].update(defaults)

# return dictionary of jobs where every job has BASE_CONFIG variables

return config

@staticmethod

def _create_jobs(modules):

"""

jobs = []

for module in modules:

for name in module.config:

# register a new job

conf = module.config[name]

try:

job = module.Service(configuration=conf, name=name)

except Exception as e:

msg.error(module.__name__ +

("/" + str(name) if name is not None else "") +

": cannot start job: '" +

str(e))

return None

else:

# set chart_name (needed to plot run time graphs)

job.chart_name = module.__name__

if name is not None:

job.chart_name += "_" + name

jobs.append(job)

msg.debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")

return [j for j in jobs if j is not None]

def _stop(self, job, reason=None):

"""

prefix = job.__module__

if job.name is not None and len(job.name) != 0:

prefix += "/" + job.name

try:

self.jobs.remove(job)

msg.info("Disabled", prefix)

except Exception as e:

msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs) + " ERROR: " + str(e))

# TODO remove section below and remove `reason`.

prefix += ": "

if reason is None:

return

elif reason[:3] == "no ":

msg.error(prefix +

"does not seem to have " +

reason[3:] +

"() function. Disabling it.")

elif reason[:7] == "failed ":

msg.error(prefix +

reason[7:] +

"() function reports failure.")

elif reason[:13] == "configuration":

msg.error(prefix +

"configuration file '" +

self.configs +

job.__module__ +

".conf' not found. Using defaults.")

elif reason[:11] == "misbehaving":

msg.error(prefix + "is " + reason)

def check(self):

"""

i = 0

overridden = []

msg.debug("all job objects", str(self.jobs))

while i < len(self.jobs):

job = self.jobs[i]

try:

if not job.check():

msg.error(job.chart_name, "check function failed.")

self._stop(job)

else:

msg.debug(job.chart_name, "check succeeded")

i += 1

try:

if job.override_name is not None:

new_name = job.__module__ + '_' + job.override_name

if new_name in overridden:

msg.error(job.override_name + " already exists. Stopping '" + job.name + "'")

self._stop(job)

i -= 1

else:

job.name = job.override_name

msg.debug(job.chart_name + " changing chart name to: '" + new_name + "'")

job.chart_name = new_name

overridden.append(job.chart_name)

except Exception:

pass

except AttributeError as e:

self._stop(job)

msg.error(job.chart_name, "cannot find check() function or it thrown unhandled exception.")

msg.debug(str(e))

except (UnboundLocalError, Exception) as e:

msg.error(job.chart_name, str(e))

self._stop(job)

msg.debug("overridden job names:", str(overridden))

msg.debug("all remaining job objects:", str(self.jobs))

def create(self):

"""

i = 0

while i < len(self.jobs):

job = self.jobs[i]

try:

if not job.create():

msg.error(job.chart_name, "create function failed.")

self._stop(job)

else:

chart = job.chart_name

sys.stdout.write(

"CHART netdata.plugin_pythond_" +

chart +

" '' 'Execution time for " +

chart +

" plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +

str(job.timetable['freq']) +

'\n')

sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")

msg.debug("created charts for", job.chart_name)

# sys.stdout.flush()

i += 1

except AttributeError:

msg.error(job.chart_name, "cannot find create() function or it thrown unhandled exception.")

self._stop(job)

except (UnboundLocalError, Exception) as e:

msg.error(job.chart_name, str(e))

self._stop(job)

def update(self):

"""

for job in self.jobs:

job.start()

while True:

if threading.active_count() <= 1:

msg.fatal("no more jobs")

time.sleep(1)

def read_config(path):

"""

try:

with open(path, 'r') as stream:

config = yaml.load(stream)

except (OSError, IOError):

msg.error(str(path), "is not a valid configuration file")

return None

except yaml.YAMLError as e:

msg.error(str(path), "is malformed:", e)

return None

return config

def parse_cmdline(directory, *commands):

"""

global DEBUG_FLAG

global OVERRIDE_UPDATE_EVERY

global BASE_CONFIG

changed_update = False

mods = []

for cmd in commands[1:]:

if cmd == "check":

pass

elif cmd == "debug" or cmd == "all":

DEBUG_FLAG = True

# redirect stderr to stdout?

elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):

#DEBUG_FLAG = True

mods.append(cmd.replace(".chart.py", ""))

else:

try:

BASE_CONFIG['update_every'] = int(cmd)

changed_update = True

except ValueError:

pass

if changed_update and DEBUG_FLAG:

OVERRIDE_UPDATE_EVERY = True

msg.debug(PROGRAM, "overriding update interval to", str(BASE_CONFIG['update_every']))

msg.debug("started from", commands[0], "with options:", *commands[1:])

return mods

def run():

"""

global DEBUG_FLAG, BASE_CONFIG

# read configuration file

disabled = []

configfile = CONFIG_DIR + "python.d.conf"

msg.PROGRAM = PROGRAM

msg.info("reading configuration file:", configfile)

log_counter = 200

log_interval = 3600

conf = read_config(configfile)

if conf is not None:

try:

# exit the whole plugin when 'enabled: no' is set in 'python.d.conf'

if conf['enabled'] is False:

msg.fatal('disabled in configuration file.\n')

except (KeyError, TypeError):

pass

try:

for param in BASE_CONFIG:

BASE_CONFIG[param] = conf[param]

except (KeyError, TypeError):

pass # use default update_every from NETDATA_UPDATE_EVERY

try:

DEBUG_FLAG = conf['debug']

except (KeyError, TypeError):

pass

try:

log_counter = conf['logs_per_interval']

except (KeyError, TypeError):

pass

try:

log_interval = conf['log_interval']

except (KeyError, TypeError):

pass

for k, v in conf.items():

if k in ("update_every", "debug", "enabled"):

continue

if v is False:

disabled.append(k)

# parse passed command line arguments

modules = parse_cmdline(MODULES_DIR, *sys.argv)

msg.DEBUG_FLAG = DEBUG_FLAG

msg.LOG_COUNTER = log_counter

msg.LOG_INTERVAL = log_interval

msg.info("MODULES_DIR='" + MODULES_DIR +

"', CONFIG_DIR='" + CONFIG_DIR +

"', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +

", ONLY_MODULES=" + str(modules))

# run plugins

charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)

charts.check()

charts.create()

charts.update()

msg.fatal("finished")

if __name__ == '__main__':

run()