commit 6cefac3c6560487abad80498240545e6835e503f Author: lmchilton Date: Wed Jun 19 10:19:04 2024 +0200 Updated pcp2openmetrics tool to fix issues: RHEL-39159, RHEL-39132, RHEL-32983, RHEL-39293. Added command line option to exclude # PCP 5 comment line Updated output to follow openmetrics format to include _total metric name ending Omitted blank line after each metric output to adhere to prometheus openmetrics parser Added # EOF line to end of text dump to adhere to prometheus openmetrics parser Updated relevent qa for pcp2openmetric changes (1131 & 1827) Updated pcp2openmetrics man page and fixed style errors(2) diff --git a/qa/1131 b/qa/1131 index 413da6fe3..aa1e554fe 100755 --- a/qa/1131 +++ b/qa/1131 @@ -62,9 +62,10 @@ pcp2json -a $A -H -I -Z UTC+0 -x "" | _archive_filter echo "---" pcp2json -a $A -H -I -z -X -b GB -P 2 -F $tmp.outfile "" echo "---" -pcp2openmetrics -s1 -H -z hinv.ncpu | _filter_pcp2openmetrics +pcp2openmetrics -s1 -z hinv.ncpu | _filter_pcp2openmetrics +echo "---" +pcp2openmetrics -s2 -x hinv.ncpu | _filter_pcp2openmetrics echo "---" - cat $tmp.outfile | _archive_filter which json_verify > /dev/null 2>&1 diff --git a/qa/1131.out b/qa/1131.out index 098d698a6..90cb9eaa2 100644 --- a/qa/1131.out +++ b/qa/1131.out @@ -2267,13 +2267,18 @@ QA output created by 1131 } --- --- - - +# EOF # HELP hinv_ncpu number of CPUs in the system # PCP5 hinv_ncpu 60.0.32 u32 PM_INDOM_NULL discrete # TYPE hinv_ncpu gauge hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} NCPU --- +# EOF +# HELP hinv_ncpu number of CPUs in the system +# TYPE hinv_ncpu gauge +hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} NCPU +hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} NCPU +--- { "@pcp": { "@hosts": [ diff --git a/qa/1827 b/qa/1827 index 47ee66d54..ebeaad460 100755 --- a/qa/1827 +++ b/qa/1827 @@ -63,12 +63,20 @@ $PCP_PYTHON_PROG $here/src/pythonserver.py $port >$tmp.python.out 2>&1 & pid=$! sleep 2 # let server start up +echo "---" echo "pcp2openmetrics invocation" | tee -a $here/$seq.full pcp2openmetrics -s1 -u http://localhost:$port hinv.ncpu >$tmp.openmetrics.out 2>$tmp.openmetrics.err echo "pcp2openmetrics HTTP POST (sorted):" _filter_pcp2openmetrics_http <$tmp.python.out +cat /dev/null > $tmp.python.out + +echo "---" +echo "testing -x option" | tee -a $here/$seq.full +pcp2openmetrics -s1 -x -u http://localhost:$port hinv.ncpu >$tmp.openmetrics2.out 2>$tmp.openmetrics2.err +_filter_pcp2openmetrics_http <$tmp.python.out + # terminate pythonserver.py now pmsignal $pid >/dev/null 2>&1 diff --git a/qa/1827.out b/qa/1827.out index e786d8607..191ebcfa8 100644 --- a/qa/1827.out +++ b/qa/1827.out @@ -1,4 +1,5 @@ QA output created by 1827 +--- pcp2openmetrics invocation pcp2openmetrics HTTP POST (sorted): @@ -21,3 +22,22 @@ INFO:root:Starting httpd... Path: / User-Agent: python-requests VERSION hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} N (cpus) +--- +testing -x option + + + + + +# HELP hinv_ncpu number of CPUs in the system +# TYPE hinv_ncpu gauge +Accept: */* +Body: +Content-Length: SIZE +Content-Type: application/openmetrics-text +Headers: +Host: localhost:PORT +INFO:root:POST request, +Path: / +User-Agent: python-requests VERSION +hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} N (cpus) diff --git a/src/pcp2openmetrics/pcp2openmetrics.1 b/src/pcp2openmetrics/pcp2openmetrics.1 index 84e81e7af..41e4e2f9c 100644 --- a/src/pcp2openmetrics/pcp2openmetrics.1 +++ b/src/pcp2openmetrics/pcp2openmetrics.1 @@ -181,6 +181,13 @@ Maximum time (in seconds) when sending a HTTP POST. Corresponding command line option is \fB\-o\fP. Defaults to \fB2.5\fP seconds. .RE +.PP +no-comment (boolean) +.RS 4 +Omit # PCP5 comment line. Omits header for some metric +data such as type, instance domain, and semantics. +Corresponding command line option is \fB\-x\fP. +Defaults to \fBno\fP. .SH OPTIONS The available command line options are: .TP 5 @@ -343,9 +350,6 @@ on .IR host , rather than from the default localhost. .TP -\fB\-H\fR, \fB\-\-no\-header\fR -Do not print any headers. -.TP \fB\-i\fR \fIinstances\fR, \fB\-\-instances\fR=\fIinstances\fR Retrieve and report only the specified metric .IR instances . @@ -605,8 +609,8 @@ and \fB\-V\fR, \fB\-\-version\fR Display version number and exit. .TP -\fB\-x\fR, \fB\-\-with\-extended\fR -Write extended information. +\fB\-x\fR, \fB\-no\-comment\fR +Omit # PCP5 comment line .TP \fB\-X\fR, \fB\-\-with\-everything\fR Write everything known about metrics, including PCP internal IDs. diff --git a/src/pcp2openmetrics/pcp2openmetrics.py b/src/pcp2openmetrics/pcp2openmetrics.py index 51b3326d7..d5a3346fe 100755 --- a/src/pcp2openmetrics/pcp2openmetrics.py +++ b/src/pcp2openmetrics/pcp2openmetrics.py @@ -50,24 +50,24 @@ class PCP2OPENMETRICS(object): self.opts = self.options() # Configuration directives - self.keys = ('source', 'output', 'derived', 'header', 'globals', - 'samples', 'interval', 'type', 'precision', 'daemonize', + self.keys = ('source', 'output', 'derived', 'globals', + 'samples', 'interval', 'precision', 'daemonize', 'timefmt', 'everything', 'count_scale', 'space_scale', 'time_scale', 'version', 'count_scale_force', 'space_scale_force', 'time_scale_force', - 'type_prefer', 'precision_force', 'limit_filter', 'limit_filter_force', + 'precision_force', 'limit_filter', 'limit_filter_force', 'live_filter', 'rank', 'invert_filter', 'predicate', 'names_change', 'speclocal', 'instances', 'ignore_incompat', 'ignore_unknown', 'omit_flat', 'include_labels', 'url', 'http_user', 'http_pass', - 'http_timeout') + 'http_timeout', 'no_comment') # Ignored for pmrep(1) compatibility self.keys_ignore = ( - 'timestamp', 'unitinfo', 'colxrow', 'separate_header', 'fixed_header', + 'timestamp','header', 'unitinfo', 'colxrow', 'separate_header', 'fixed_header', 'delay', 'width', 'delimiter', 'extcsv', 'width_force', 'extheader', 'repeat_header', 'interpol', 'dynamic_header', 'overall_rank', 'overall_rank_alt', 'sort_metric', - 'instinfo', 'include_texts') + 'instinfo', 'include_texts', 'type', 'type_prefer') # The order of preference for options (as present): # 1 - command line options @@ -79,13 +79,12 @@ class PCP2OPENMETRICS(object): self.output = None # For pmrep conf file compat only self.speclocal = None self.derived = None - self.header = 1 self.globals = 1 self.samples = None # forever self.interval = pmapi.timeval(10) # 10 sec self.opts.pmSetOptionInterval(str(10)) # 10 sec self.delay = 0 - self.type = 0 + self.type = 1 self.type_prefer = self.type self.ignore_incompat = 0 self.ignore_unknown = 0 @@ -118,6 +117,8 @@ class PCP2OPENMETRICS(object): self.http_user = None self.http_pass = None self.http_timeout = TIMEOUT + self.no_comment = False + self.header_flag = True # Internal self.runtime = -1 @@ -146,7 +147,7 @@ class PCP2OPENMETRICS(object): opts = pmapi.pmOptions() opts.pmSetOptionCallback(self.option) opts.pmSetOverrideCallback(self.option_override) - opts.pmSetShortOptions("a:h:LK:c:Ce:D:V?HGA:S:T:O:s:t:rRIi:jJ:4:58:9:nN:vmP:0:q:b:y:Q:B:Y:F:f:Z:zXo:p:U:u:") + opts.pmSetShortOptions("a:h:LK:c:Ce:D:V?GA:S:T:O:s:t:Ii:jJ:4:58:9:nN:vmP:0:q:b:y:Q:B:Y:F:f:Z:zXo:p:U:u:x") opts.pmSetShortUsage("[option...] metricspec [...]") opts.pmSetLongOptionHeader("General options") @@ -166,7 +167,6 @@ class PCP2OPENMETRICS(object): opts.pmSetLongOptionHelp() # -?/--help opts.pmSetLongOptionHeader("Reporting options") - opts.pmSetLongOption("no-header", 0, "H", "", "omit headers") opts.pmSetLongOption("no-globals", 0, "G", "", "omit global metrics") opts.pmSetLongOptionAlign() # -A/--align opts.pmSetLongOptionStart() # -S/--start @@ -176,8 +176,6 @@ class PCP2OPENMETRICS(object): opts.pmSetLongOptionInterval() # -t/--interval opts.pmSetLongOptionTimeZone() # -Z/--timezone opts.pmSetLongOptionHostZone() # -z/--hostzone - opts.pmSetLongOption("raw", 0, "r", "", "output raw counter values (no rate conversion)") - opts.pmSetLongOption("raw-prefer", 0, "R", "", "prefer output raw counter values (no rate conversion)") opts.pmSetLongOption("ignore-incompat", 0, "I", "", "ignore incompatible instances (default: abort)") opts.pmSetLongOption("ignore-unknown", 0, "5", "", "ignore unknown metrics (default: abort)") opts.pmSetLongOption("names-change", 1, "4", "ACTION", "update/ignore/abort on PMNS change (default: ignore)") @@ -201,6 +199,7 @@ class PCP2OPENMETRICS(object): opts.pmSetLongOption("time-scale-force", 1, "Y", "SCALE", "forced time unit") opts.pmSetLongOption("with-everything", 0, "X", "", "write everything, incl. internal IDs") + opts.pmSetLongOption("no-comment", 0, "x", "", "omit comment lines") opts.pmSetLongOption("url", 1, "u", "URL", "URL of endpoint to receive HTTP POST") opts.pmSetLongOption("http-timeout", 1, "o", "SECONDS", "timeout when sending HTTP POST") @@ -238,14 +237,8 @@ class PCP2OPENMETRICS(object): self.derived = ";" + optarg else: self.derived = self.derived + ";" + optarg - elif opt == 'H': - self.header = 0 elif opt == 'G': self.globals = 0 - elif opt == 'r': - self.type = 1 - elif opt == 'R': - self.type_prefer = 1 elif opt == 'I': self.ignore_incompat = 1 elif opt == '5': @@ -298,6 +291,8 @@ class PCP2OPENMETRICS(object): self.time_scale_force = optarg elif opt == 'X': self.everything = 1 + elif opt == 'x': + self.no_comment = True elif opt == 'u': self.url = optarg elif opt == 'o': @@ -319,6 +314,7 @@ class PCP2OPENMETRICS(object): if pmapi.c_api.pmSetContextOptions(self.context.ctx, self.opts.mode, self.opts.delta): raise pmapi.pmUsageErr() + self.context.pmNewZone("UTC") def validate_config(self): """ Validate configuration options """ @@ -346,11 +342,6 @@ class PCP2OPENMETRICS(object): # Common preparations self.context.prepare_execute(self.opts, False, self.interpol, self.interval) - # Headers - if self.header == 1: - self.header = 0 - self.write_header() - # Just checking if self.check == 1: return @@ -394,29 +385,12 @@ class PCP2OPENMETRICS(object): self.write_openmetrics(tstamp) - def write_header(self): - """ Write info header """ - output = self.outfile if self.outfile else "stdout" - if self.context.type == PM_CONTEXT_ARCHIVE: - sys.stdout.write('"# Writing %d archived metrics to %s..." }\n{ "//": "(Ctrl-C to stop)" }\n' % (len(self.metrics), output)) - return - - sys.stdout.write('# "Waiting for %d metrics to be written to %s' % (len(self.metrics), output)) - if self.runtime != -1: - sys.stdout.write('\n # "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), self.runtime)) - elif self.samples: - duration = (self.samples - 1) * float(self.interval) - sys.stdout.write('\n # "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), duration)) - else: - sys.stdout.write('...\n# "(Ctrl-C to stop)" }\n') - def write_openmetrics(self, timestamp): """ Write results in openmetrics format """ if timestamp is None: # Silent goodbye, close in finalize() return - self.context.pmNewZone("UTC") ts = self.context.datetime_to_secs(self.pmfg_ts(), PM_TIME_SEC) if self.prev_ts is None: @@ -500,11 +474,12 @@ class PCP2OPENMETRICS(object): help_dict = {} help_dict[metric] = context.pmLookupText(pmid[0]) - if self.context.type == PM_CONTEXT_ARCHIVE: - body += "\nMetric %s details (last fetch: %d)\n:" % (metric, ts) - body += '# PCP5 %s %s %s %s %s %s\n' % (openmetrics_name(metric), pmIDStr, get_type_string(desc), pmIndomStr, semantics, units) - body += '# HELP %s %s\n' % (openmetrics_name(metric), help_dict[metric]) - body += '# TYPE %s %s\n' % (openmetrics_name(metric), openmetrics_type(desc)) + if self.header_flag is True: + if self.no_comment is False: + body += '# PCP5 %s %s %s %s %s %s\n' % (openmetrics_name(metric), pmIDStr, get_type_string(desc), pmIndomStr, semantics, units) + body += '# TYPE %s %s\n' % (openmetrics_name(metric), openmetrics_type(desc)) + body += '# HELP %s %s\n' % (openmetrics_name(metric), help_dict[metric]) + self.header_flag = False for inst, name, value in results[metric]: if isinstance(value, float): @@ -515,10 +490,16 @@ class PCP2OPENMETRICS(object): value = format(value, fmt) else: str(value) + + if openmetrics_type(desc) == "counter": + openmetrics_name_end = openmetrics_name(metric) + "_total" + else: + openmetrics_name_end = openmetrics_name(metric) + if self.context.type == PM_CONTEXT_ARCHIVE: - body += '%s%s %s %s\n' % (openmetrics_name(metric), openmetrics_labels(inst, name, desc, labels), value, ts) + body += '%s%s %s %s\n' % (openmetrics_name_end, openmetrics_labels(inst, name, desc, labels), value, ts) else: - body += '%s%s %s\n' % (openmetrics_name(metric), openmetrics_labels(inst, name, desc, labels), value) + body += '%s%s %s\n' % (openmetrics_name_end, openmetrics_labels(inst, name, desc, labels), value) if self.url: auth = None @@ -537,13 +518,13 @@ class PCP2OPENMETRICS(object): elif self.outfile: self.writer.write(body) else: - print(body) + sys.stdout.write(body) def finalize(self): """ Finalize and clean up """ if self.writer: try: - self.writer.write("\n") + self.writer.write("# EOF\n") self.writer.flush() except IOError as write_error: if write_error.errno != errno.EPIPE: