Current File : //usr/lib/fm-agent/library/plugin_manager.py
from agent_util import Plugin
import logging
import os
import sys
import traceback
import tempfile
import tarfile
import zipfile
try:
from hashlib import sha1
except:
from sha import sha as sha1
try:
# Python 2.x
import urllib2
except:
import urllib.request as urllib2
# In case of python 3
try:
import ConfigParser as configparser
except:
import configparser
def execfile3(filepath, globals=None, locals=None):
if globals is None:
globals = {}
globals.update(
{
"__file__": filepath,
"__name__": "__main__",
}
)
f = open(filepath, "rb")
exec(compile(f.read(), filepath, "exec"), globals, locals)
f.close()
class PluginManager(object):
def __init__(self, db, config_file, *directories):
self.log = logging.getLogger(self.__class__.__name__)
self.db = db
self.config_file = config_file
self.directories = directories
self.plugins = self._load_plugins(directories)
# Internal container for metadata so we can avoid repeatedly regenerating it
self._metadata = None
self._hashed_metadata = None
# now we load all the plugin-specific configs and map them to
# self.config
self.config = {}
if os.path.exists(self.config_file):
config_file = configparser.ConfigParser()
config_file.read(self.config_file)
for textkey in config_file.sections():
try:
debug = config_file.get("agent", "debug")
except:
debug = False
self.config[textkey] = {"debug": debug}
for k, v in config_file.items(textkey):
self.config[textkey][k] = v
def add_dem_wifi_results(self, wifi_results):
dem_config = self.config.get("dem", None)
if dem_config:
dem_config["wifi_results"] = wifi_results
def install_remote_plugins(self, url):
"""
Grab the remote url and extract the contents, putting them in the
appropiate plugin folder.
"""
base_custom_plugin_dir = self.directories[-1]
f = tempfile.NamedTemporaryFile(delete=False)
num_installed = 0
ext = url.split(".")[-1]
fname = url.split("/")[-1]
try:
r = urllib2.urlopen(url)
if "content-disposition" in r.info():
ext = r.info().getheader("content-disposition").split(".")[-1]
f.write(r.read())
f.close()
except Exception:
self.log.error("Unable to download URL: %s" % traceback.format_exc())
return
if ext in ("tar", "tgz"):
try:
t = tarfile.open(f.name)
for file in t.getnames():
if file.endswith(".py"):
self.log.info(" Installing plugin %s" % file)
t.extract(file, base_custom_plugin_dir)
num_installed += 1
except Exception:
self.log.error(
"Unable to extract tar contents: %s" % traceback.format_exc()
)
elif ext == "zip":
try:
z = zipfile.ZipFile(f.name)
for file in z.namelist():
if file.endswith(".py"):
self.log.info(" Installing plugin %s" % file)
z.extract(file, base_custom_plugin_dir)
num_installed += 1
except Exception:
self.log.error(
"Unable to extract zip contents: %s" % traceback.format_exc()
)
elif ext == "py":
self.log.info(" Installing plugin %s" % fname)
os.system(
"cp %s %s" % (f.name, os.path.join(base_custom_plugin_dir, fname))
)
num_installed += 1
else:
self.log.error(
"Unable to install Agent Plugin, unknown extension: %s" % ext
)
if num_installed:
self.log.info("\nInstalled %s Agent plugins" % num_installed)
os.system("rm -f %s" % f.name)
# run a specific plugin
def check(self, schedule):
try:
self_key = self.config.get("agent", {}).get("server_key")
container = None
if (
getattr(schedule, "server_key", None)
and schedule.server_key != self_key
):
container_id = schedule.server_key.split(":")[-1]
try:
container = self.db["docker_containers"][container_id]
except:
if (
schedule.plugin_textkey == "docker"
and schedule.resource_textkey == "status.running"
):
# Special handling of "container is running" metrics.
# Pass in a phony value to treat it as gone.
container = {"State": "deleted"}
else:
self.log.error(
"Could not find container with id %s", container_id
)
return None
plugin = self.plugins[schedule.plugin_textkey](schedule)
config = self.config.get(schedule.plugin_textkey, {})
scale = config.get("scale", 1.0)
if container:
value = plugin.check_docker(
container, schedule.resource_textkey, schedule.option, config
)
else:
value = plugin.check(schedule.resource_textkey, schedule.option, config)
if value is not None:
value *= scale
except Exception:
self.log.exception(
"%s/%s check failed: %s"
% (
schedule.plugin_textkey,
schedule.resource_textkey,
traceback.format_exc(),
)
)
return None
return value
def _load_plugins(self, directories):
plugins = {}
plugin_blacklist = ""
if os.path.exists(self.config_file):
config_file = configparser.ConfigParser()
config_file.read(self.config_file)
try:
plugin_blacklist = config_file.get("agent", "plugin_blacklist")
except:
plugin_blacklist = ""
if plugin_blacklist:
plugin_blacklist = list(map(str.strip, plugin_blacklist.split(",")))
def plugin_file_checker():
for plugin_dir in directories:
self.log.info("Looking in directory %r", plugin_dir)
for name in os.listdir(plugin_dir):
try:
if name[-3:] != ".py" or name == "__init__.py":
continue
plugin_file = os.path.join(plugin_dir, name)
# We've registed sys.modules['agent'] to equal THIS
# module, so this exec has the effect of 'importing' the
# plugin, and resolving 'import agent' to import this
# module. This is so we never have to change the plugins
# in order to whitelabel.
f_globals = {}
f_locals = {}
if sys.version_info[0] == 3:
execfile3(plugin_file, f_globals, f_globals)
else:
execfile(plugin_file, f_globals, f_globals)
for obj in list(f_globals.values()):
yield obj
self.log.debug("Loaded %r", plugin_file)
except Exception:
t, e = sys.exc_info()[:2]
self.log.info("Error importing plugin %s, skipping", name)
self.log.debug(e)
continue
self.log.info("Loaded %d plugins", len(plugins))
for obj in plugin_file_checker():
if isinstance(obj, type):
if (
issubclass(obj, Plugin)
and hasattr(obj, "textkey")
and obj.textkey not in plugin_blacklist
):
if obj.textkey != "TEMPLATE_PLUGIN":
plugins[obj.textkey] = obj
return plugins
# returns all the values needed to describe the plugins and their resources
# in a json-serializable way. Uses local cache of values to avoid recomputing.
@property
def metadata(self):
if self._metadata == None:
# We haven't previously built the metadata - go do that and cache for later
metadata = {}
for plugin in list(self.plugins.values()):
# get the config and push it up with the metadata
config = self.config.get(plugin.textkey, {})
try:
meta = format_metadata(plugin.get_metadata(config))
except:
self.log.exception("error getting metadata from %r", plugin.textkey)
continue
self.log.info(
"got %d textkeys for %r", len(list(meta.keys())), plugin.textkey
)
label = getattr(plugin, "label", plugin.textkey)
metadata[plugin.textkey] = (label, meta)
self._metadata = metadata
return self._metadata
# returns the metadata hash, used for comparing whether or not our metadata
# has changed since we last pushed. Uses local cache of values to avoid recomputing.
def hashed_metadata(self):
if self._hashed_metadata == None:
flat_data = []
for textkey, (label, plugin_metadata) in list(self.metadata.items()):
plugin_options = []
for resource_name, resource_metadata in list(plugin_metadata.items()):
resource_meta = sorted(repr(resource_metadata))
plugin_options.append(repr((resource_name, resource_metadata)))
plugin_options.sort()
flat_data.append(repr((textkey, plugin_options, label)))
flat_data.sort()
self._hashed_metadata = sha1(repr(flat_data).encode()).hexdigest()
return self._hashed_metadata
def is_metadata_stale(self):
self.log.info(
"comparing metadata hashes %r with %r",
self.hashed_metadata(),
self.db["last_metadata"],
)
return self.db["last_metadata"] != self.hashed_metadata()
def format_metadata(metadata):
"""
Go over the metadata entry and make sure the option resources
are matching to the new style of dictionaries if they
are a string.
"""
for resource, entry in metadata.items():
options = entry["options"]
new_options = []
if not options:
continue
for option in options:
if sys.version_info[0] == 3:
if (
type(option) == str
): # TODO Verify that unicode is present on old python installs.
new_options.append({"resource": option})
else:
if type(option) in (
str,
unicode,
): # TODO Verify that unicode is present on old python installs.
new_options.append({"resource": option})
if new_options:
entry["options"] = new_options
if not entry.get("options_schema"):
entry["options_schema"] = {"resource": "string"}
return metadata
Mini Shell