ok
Mini Shell
import agent_util
from plugins.process import ProcessPlugin
try:
import ssl
except:
ssl = None
try:
# Python 2.x
from httplib import HTTPConnection, HTTPSConnection
except:
from http.client import HTTPConnection, HTTPSConnection
from library.log_matcher import LogMatcher
import traceback
# ON FREEBSD/CENTOS, THEY MAY NEED TO ADD THIS TO THEIR HTTPD.CONF/APACHE2.CONF:
# LoadModule status_module libexec/apache22/mod_status.so
# <IfModule status_module>
# ExtendedStatus On
# <Location /server-status>
# SetHandler server-status
# Order deny,allow
# Allow from all
# </Location>
# </IfModule>
class ApachePlugin(agent_util.Plugin):
textkey = "apache"
label = "Apache Webserver"
DEFAULTS = {
"server_status_protocol": "http",
"server_status_host": "localhost",
"server_status_url": "server-status",
"apache_log_files": [
"/var/log/apache2/access.log",
"/var/log/httpd/access.log",
"/var/log/httpd-access.log",
],
}
LOG_COUNT_EXPRESSIONS = {
"apache.4xx": r"4\d{2}",
"apache.5xx": r"5\d{2}",
"apache.2xx": r"2\d{2}",
}
@classmethod
def get_data(self, textkey, ip, config):
server_status_path = ""
server_status_url = config.get("server_status_url")
server_status_protocol = config.get("server_status_protocol")
server_status_port = config.get("server_status_port", None)
if not server_status_url.startswith("/"):
server_status_path += "/"
server_status_path += server_status_url + "?auto"
if server_status_protocol == "https" and ssl is not None:
if server_status_port is None:
conn = HTTPSConnection(ip, context=ssl._create_unverified_context())
else:
conn = HTTPSConnection(
ip,
int(server_status_port),
context=ssl._create_unverified_context(),
)
else:
if server_status_port:
conn = HTTPConnection(ip, server_status_port)
else:
conn = HTTPConnection(ip)
try:
conn.request("GET", server_status_path)
r = conn.getresponse()
output = r.read().decode()
conn.close()
except:
self.log.info(
"""
Unable to access the Apache status page at %s%s. Please ensure Apache is running and
the server status url is correctly specified.
"""
% (ip, server_status_path)
)
self.log.info("error: %s" % traceback.format_exc())
return None
data = dict()
for line in output.splitlines():
if ":" not in line:
continue
k, v = line.split(": ", 1)
data.update({k: v})
def get_param_value(data, param, output_type):
try:
return output_type(data[param])
except KeyError:
return None
if textkey.endswith("uptime"):
return get_param_value(data, "Uptime", int)
elif textkey.endswith("total_accesses"):
return get_param_value(data, "Total Accesses", int)
elif textkey.endswith("total_traffic"):
val = get_param_value(data, "Total kBytes", int)
# Convert to MB for backwards compatibility
if val:
return val / 1000.0
else:
return None
elif textkey.endswith("cpu_load"):
return get_param_value(data, "CPULoad", float)
elif textkey.endswith("connections"):
return get_param_value(data, "ReqPerSec", float)
elif textkey.endswith("transfer_rate"):
val = get_param_value(data, "BytesPerSec", float)
# Convert to MB for backwards compatibility
return val / (1000.0**2)
elif textkey.endswith("avg_request_size"):
val = get_param_value(data, "BytesPerReq", float)
# Convert to MB for backwards compatibility
return val / (1000.0**2)
elif textkey.endswith("workers_used_count"):
return get_param_value(data, "BusyWorkers", int)
elif textkey.endswith("workers_idle_count"):
return get_param_value(data, "IdleWorkers", int)
elif textkey in ("apache.workers_used", "apache.workers_idle"):
busy = get_param_value(data, "BusyWorkers", int)
idle = get_param_value(data, "IdleWorkers", int)
if busy is None or idle is None:
return None
total = busy + idle
if textkey.endswith("workers_used"):
return float(100.0 * busy / total)
elif textkey.endswith("workers_idle"):
return float(100.0 * idle / total)
@classmethod
def get_metadata(self, config):
status = agent_util.SUPPORTED
msg = None
self.log.info("Looking for apache2ctl to confirm apache is installed")
# look for either overrides on how to access the apache healthpoint or one of the apachectl bins
if (
not agent_util.which("apache2ctl")
and not agent_util.which("apachectl")
and not config.get("from_docker")
and not agent_util.which("httpd")
):
self.log.info("Couldn't find apachectl or apache2ctl")
status = agent_util.UNSUPPORTED
msg = "Apache wasn't detected (apachectl or apache2ctl)"
return {}
# update default config with anything provided in the config file
if config:
new_config = self.DEFAULTS.copy()
new_config.update(config)
config = new_config
# Look for Apache server-status endpoint
server_status_path = ""
server_status_protocol = config.get(
"server_status_protocol", self.DEFAULTS["server_status_protocol"]
)
server_status_url = config.get(
"server_status_url", self.DEFAULTS["server_status_url"]
)
server_status_port = config.get("server_status_port", None)
if not server_status_url.startswith("/"):
server_status_path += "/"
server_status_path += server_status_url + "?auto"
not_found_error = (
"""
Unable to access the Apache status page at %s. Please ensure the status page module is enabled,
Apache is running, and, optionally, the server status url is correctly specified. See docs.fortimonitor.forticloud.com/
for more information.
"""
% server_status_path
)
host_list = []
# support optional comma delimitted addresses
ip_list = config.get("server_status_host", "localhost")
ip_list = ip_list.split(",")
# loop over each IP from config and check to see if the status endpoint is reachable
for ip in ip_list:
ip_working = True
try:
if server_status_protocol == "https" and ssl is not None:
if server_status_port is None:
conn = HTTPSConnection(
ip, context=ssl._create_unverified_context()
)
else:
conn = HTTPSConnection(
ip,
int(server_status_port),
context=ssl._create_unverified_context(),
)
else:
if server_status_port:
conn = HTTPConnection(ip, server_status_port)
else:
conn = HTTPConnection(ip)
conn.request("GET", server_status_path)
r = conn.getresponse()
conn.close()
except Exception:
import sys
_, err_msg, _ = sys.exc_info()
ip_working = False
self.log.info(not_found_error)
self.log.info("error: %s" % err_msg)
msg = not_found_error
continue
if r.status != 200:
self.log.info(not_found_error)
msg = not_found_error
ip_working = False
if ip_working:
host_list.append(ip)
output = r.read()
if config.get("debug", False):
self.log.info(
"#####################################################"
)
self.log.info("Apache server-status output:")
self.log.info(output)
self.log.info(
"#####################################################"
)
if not host_list:
status = agent_util.MISCONFIGURED
msg = not_found_error
return {}
# Checking log files access
if not config.get("apache_log_files"):
log_files = self.DEFAULTS.get("apache_log_files")
else:
log_files = config.get("apache_log_files")
try:
if type(log_files) in (str, unicode):
log_files = log_files.split(",")
except NameError:
if type(log_files) in (str, bytes):
log_files = log_files.split(",")
can_access = False
log_file_msg = ""
log_file_status = status
for log_file in log_files:
try:
opened = open(log_file, "r")
opened.close()
# Can access at least one file. Support log access
can_access = True
except Exception:
import sys
_, error, _ = sys.exc_info()
message = (
"Error opening the file %s. Ensure the fm-agent user has access to read this file"
% log_file
)
if "Permission denied" in str(error):
self.log.error(error)
self.log.error(message)
if log_file not in self.DEFAULTS.get("apache_log_files", []):
self.log.error(error)
self.log.error(message)
log_file_msg = message
log_file_status = agent_util.MISCONFIGURED
if can_access:
log_file_status = agent_util.SUPPORTED
log_file_msg = ""
metadata = {
"apache.workers_used": {
"label": "Workers - percent serving requests",
"options": host_list,
"status": status,
"error_message": msg,
"unit": "%",
},
"apache.workers_idle": {
"label": "Workers - percent idle",
"options": host_list,
"status": status,
"error_message": msg,
"unit": "%",
},
"apache.workers_used_count": {
"label": "Workers - count serving requests",
"options": host_list,
"status": status,
"error_message": msg,
"unit": "workers",
},
"apache.workers_idle_count": {
"label": "Workers - count idle",
"options": host_list,
"status": status,
"error_message": msg,
"unit": "workers",
},
"apache.uptime": {
"label": "Server uptime",
"options": host_list,
"status": status,
"error_message": msg,
"unit": "seconds",
},
"apache.total_accesses": {
"label": "Request count",
"options": host_list,
"status": status,
"error_message": msg,
"unit": "requests",
},
"apache.total_traffic": {
"label": "Total content served",
"options": host_list,
"status": status,
"error_message": msg,
"unit": "MB",
},
"apache.cpu_load": {
"label": "Percentage of CPU used by all workers",
"options": host_list,
"status": status,
"error_message": msg,
"unit": "%",
},
"apache.connections": {
"label": "Requests per second",
"options": host_list,
"status": status,
"error_message": msg,
"unit": "requests",
},
"apache.transfer_rate": {
"label": "Transfer rate",
"options": host_list,
"status": status,
"error_message": msg,
"unit": "MB/s",
},
"apache.avg_request_size": {
"label": "Request size average",
"options": host_list,
"status": status,
"error_message": msg,
"unit": "MB",
},
"apache.2xx": {
"label": "Rate of 2xx's events",
"options": None,
"status": log_file_status,
"error_message": log_file_msg,
"unit": "entries/s",
},
"apache.4xx": {
"label": "Rate of 4xx's events",
"options": None,
"status": log_file_status,
"error_message": log_file_msg,
"unit": "entries/s",
},
"apache.5xx": {
"label": "Rate of 5xx's events",
"options": None,
"status": log_file_status,
"error_message": log_file_msg,
"unit": "entries/s",
},
"apache.is_running": {
"label": "Apache is running",
"options": None,
"status": status,
"error_message": msg,
},
}
return metadata
@classmethod
def get_metadata_docker(self, container, config):
if "server_status_host" not in config:
try:
ip = agent_util.get_container_ip(container)
config["server_status_host"] = ip
except Exception:
import sys
_, e, _ = sys.exc_info()
self.log.exception(e)
config["from_docker"] = True
return self.get_metadata(config)
def get_apache_process_name(self):
if agent_util.which("apache2ctl"):
return "apache2"
if agent_util.which("httpd"):
return "httpd"
if agent_util.which("httpd22"):
return "httpd22"
return None
def check(self, textkey, ip, config):
# update default config with anything provided in the config file
new_config = self.DEFAULTS.copy()
new_config.update(config)
config = new_config
# add backwards compatibility for older entries where IP was not an option. Pull from config instead.
if not ip:
ip = config["server_status_host"].split(",")[0]
if textkey == "apache.is_running" and not config.get("from_docker"):
apache_process = ProcessPlugin(None)
apache_process_name = self.get_apache_process_name()
if apache_process_name is not None:
apache_is_running = apache_process.check(
"process.exists", apache_process_name, {}
)
return apache_is_running
return None
if textkey in ("apache.2xx", "apache.4xx", "apache.5xx"):
file_inodes = {}
total_metrics = 0
timescale = 1
column = 8
expression = self.LOG_COUNT_EXPRESSIONS.get(textkey)
if not config.get("apache_log_files"):
log_files = config["apache_log_files"]
else:
log_files = config.get("apache_log_files")
try:
if type(log_files) in (str, unicode):
log_files = log_files.split(",")
except NameError:
if type(log_files) in (str, bytes):
log_files = log_files.split(",")
for target in log_files:
try:
file_inodes[target] = LogMatcher.get_file_inode(target)
except OSError:
import sys
_, error, _ = sys.exc_info()
if "Permission denied" in str(error):
self.log.error(
"Error opening the file %s. Ensure the fm-agent user has access to read this file"
% target
)
self.log.error(str(error))
if target not in self.DEFAULTS.get("apache_log_files", []):
self.log.error(str(error))
self.log.error(
"Error opening the file %s. Ensure the fm-agent user has access to read this file"
% target
)
continue
log_data = self.get_cache_results(
textkey, "%s/%s" % (self.schedule.id, target)
)
if log_data:
log_data = log_data[0][-1]
else:
log_data = dict()
last_line_number = log_data.get("last_known_line")
stored_inode = log_data.get("inode")
results = log_data.get("results", [])
try:
total_lines, current_lines = LogMatcher.get_file_lines(
last_line_number, target, file_inodes[target], stored_inode
)
except IOError:
import sys
_, error, _ = sys.exc_info()
self.log.error(
"Unable to read the file %s. Ensure the fm-agent user has access to read this file"
% target
)
continue
self.log.info(
"Stored line %s Current line %s looking at %s lines"
% (str(last_line_number), str(total_lines), str(len(current_lines)))
)
log_matcher = LogMatcher(stored_inode)
results = log_matcher.match_in_column(current_lines, expression, column)
metric, results = log_matcher.calculate_metric(results, timescale)
total_metrics += metric and metric or 0
self.log.info(
'Found %s instances of "%s" in %s'
% (str(metric or 0), expression, target)
)
previous_result = self.get_cache_results(
textkey, "%s/%s" % (self.schedule.id, target)
)
cache_data = dict(
inode=file_inodes[target],
last_known_line=total_lines,
results=results,
)
self.cache_result(
textkey,
"%s/%s" % (self.schedule.id, target),
cache_data,
replace=True,
)
if not previous_result:
return None
else:
delta, prev_data = previous_result[0]
try:
curr_count = cache_data.get("results")[0][-1]
result = curr_count / float(delta)
except IndexError:
result = None
return result
else:
return ApachePlugin.get_data(textkey, ip, config)
def check_docker(self, container, textkey, ip, config):
try:
ip = agent_util.get_container_ip(container)
except:
return None
return self.check(textkey, ip, config)
Zerion Mini Shell 1.0