1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define STATSD_DEBUG false // STOPSHIP if true
18 #include "Log.h"
19
20 #include "KllMetricProducer.h"
21
22 #include <limits.h>
23 #include <stdlib.h>
24
25 #include "guardrail/StatsdStats.h"
26 #include "metrics/parsing_utils/metrics_manager_util.h"
27 #include "stats_log_util.h"
28
29 using android::util::FIELD_COUNT_REPEATED;
30 using android::util::FIELD_TYPE_BYTES;
31 using android::util::FIELD_TYPE_INT32;
32 using android::util::FIELD_TYPE_MESSAGE;
33 using android::util::ProtoOutputStream;
34 using std::nullopt;
35 using std::optional;
36 using std::string;
37 using zetasketch::android::AggregatorStateProto;
38
39 namespace android {
40 namespace os {
41 namespace statsd {
42
43 // for StatsLogReport
44 const int FIELD_ID_KLL_METRICS = 16;
45 // for KllBucketInfo
46 const int FIELD_ID_SKETCH_INDEX = 1;
47 const int FIELD_ID_KLL_SKETCH = 2;
48 const int FIELD_ID_SKETCHES = 3;
49 const int FIELD_ID_BUCKET_NUM = 4;
50 const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 5;
51 const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 6;
52 const int FIELD_ID_CONDITION_TRUE_NS = 7;
53
KllMetricProducer(const ConfigKey & key,const KllMetric & metric,const uint64_t protoHash,const PullOptions & pullOptions,const BucketOptions & bucketOptions,const WhatOptions & whatOptions,const ConditionOptions & conditionOptions,const StateOptions & stateOptions,const ActivationOptions & activationOptions,const GuardrailOptions & guardrailOptions,const wp<ConfigMetadataProvider> configMetadataProvider)54 KllMetricProducer::KllMetricProducer(const ConfigKey& key, const KllMetric& metric,
55 const uint64_t protoHash, const PullOptions& pullOptions,
56 const BucketOptions& bucketOptions,
57 const WhatOptions& whatOptions,
58 const ConditionOptions& conditionOptions,
59 const StateOptions& stateOptions,
60 const ActivationOptions& activationOptions,
61 const GuardrailOptions& guardrailOptions,
62 const wp<ConfigMetadataProvider> configMetadataProvider)
63 : ValueMetricProducer(metric.id(), key, protoHash, pullOptions, bucketOptions, whatOptions,
64 conditionOptions, stateOptions, activationOptions, guardrailOptions,
65 configMetadataProvider) {
66 }
67
getDumpProtoFields() const68 KllMetricProducer::DumpProtoFields KllMetricProducer::getDumpProtoFields() const {
69 return {FIELD_ID_KLL_METRICS,
70 FIELD_ID_BUCKET_NUM,
71 FIELD_ID_START_BUCKET_ELAPSED_MILLIS,
72 FIELD_ID_END_BUCKET_ELAPSED_MILLIS,
73 FIELD_ID_CONDITION_TRUE_NS,
74 /*conditionCorrectionNsFieldId=*/nullopt};
75 }
76
writePastBucketAggregateToProto(const int aggIndex,const unique_ptr<KllQuantile> & kll,const int sampleSize,ProtoOutputStream * const protoOutput) const77 void KllMetricProducer::writePastBucketAggregateToProto(
78 const int aggIndex, const unique_ptr<KllQuantile>& kll, const int sampleSize,
79 ProtoOutputStream* const protoOutput) const {
80 uint64_t sketchesToken =
81 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_SKETCHES);
82 protoOutput->write(FIELD_TYPE_INT32 | FIELD_ID_SKETCH_INDEX, aggIndex);
83
84 // TODO(b/186737273): Serialize directly to ProtoOutputStream
85 const AggregatorStateProto& aggProto = kll->SerializeToProto();
86 const size_t numBytes = aggProto.ByteSizeLong();
87 const unique_ptr<char[]> buffer(new char[numBytes]);
88 aggProto.SerializeToArray(&buffer[0], numBytes);
89 protoOutput->write(FIELD_TYPE_BYTES | FIELD_ID_KLL_SKETCH, &buffer[0], numBytes);
90
91 VLOG("\t\t sketch %d: %zu bytes", aggIndex, numBytes);
92 protoOutput->end(sketchesToken);
93 }
94
getInt64ValueFromEvent(const LogEvent & event,const Matcher & matcher)95 optional<int64_t> getInt64ValueFromEvent(const LogEvent& event, const Matcher& matcher) {
96 for (const FieldValue& value : event.getValues()) {
97 if (value.mField.matches(matcher)) {
98 switch (value.mValue.type) {
99 case INT:
100 return {value.mValue.int_value};
101 case LONG:
102 return {value.mValue.long_value};
103 default:
104 return nullopt;
105 }
106 }
107 }
108 return nullopt;
109 }
110
aggregateFields(const int64_t eventTimeNs,const MetricDimensionKey & eventKey,const LogEvent & event,vector<Interval> & intervals,Empty & empty)111 bool KllMetricProducer::aggregateFields(const int64_t eventTimeNs,
112 const MetricDimensionKey& eventKey, const LogEvent& event,
113 vector<Interval>& intervals, Empty& empty) {
114 bool seenNewData = false;
115 for (size_t i = 0; i < mFieldMatchers.size(); i++) {
116 const Matcher& matcher = mFieldMatchers[i];
117 Interval& interval = intervals[i];
118 interval.aggIndex = i;
119 const optional<int64_t> valueOpt = getInt64ValueFromEvent(event, matcher);
120 if (!valueOpt) {
121 VLOG("Failed to get value %zu from event %s", i, event.ToString().c_str());
122 StatsdStats::getInstance().noteBadValueType(mMetricId);
123 return seenNewData;
124 }
125
126 // interval.aggregate can be nullptr from cases:
127 // 1. Initialization from default construction of Interval struct.
128 // 2. Ownership of the unique_ptr<KllQuantile> at interval.aggregate being transferred to
129 // PastBucket after flushing.
130 if (!interval.aggregate) {
131 interval.aggregate = KllQuantile::Create();
132 }
133 seenNewData = true;
134 interval.aggregate->Add(valueOpt.value());
135 interval.sampleSize += 1;
136 }
137 return seenNewData;
138 }
139
buildPartialBucket(int64_t bucketEndTimeNs,vector<Interval> & intervals)140 PastBucket<unique_ptr<KllQuantile>> KllMetricProducer::buildPartialBucket(
141 int64_t bucketEndTimeNs, vector<Interval>& intervals) {
142 PastBucket<unique_ptr<KllQuantile>> bucket;
143 bucket.mBucketStartNs = mCurrentBucketStartTimeNs;
144 bucket.mBucketEndNs = bucketEndTimeNs;
145 for (Interval& interval : intervals) {
146 if (interval.hasValue()) {
147 bucket.aggIndex.push_back(interval.aggIndex);
148 // Transfer ownership of unique_ptr<KllQuantile> from interval.aggregate to
149 // bucket.aggregates vector. interval.aggregate is guaranteed to be nullptr after this.
150 bucket.aggregates.push_back(std::move(interval.aggregate));
151 }
152 }
153 return bucket;
154 }
155
156 // Estimate for the size of NumericValues.
getAggregatedValueSize(const std::unique_ptr<KllQuantile> & kll) const157 size_t KllMetricProducer::getAggregatedValueSize(const std::unique_ptr<KllQuantile>& kll) const {
158 size_t valueSize = 0;
159 // Index
160 valueSize += sizeof(int32_t);
161
162 // Value
163 valueSize += kll->SerializeToProto().ByteSizeLong();
164
165 return valueSize;
166 }
167
byteSizeLocked() const168 size_t KllMetricProducer::byteSizeLocked() const {
169 sp<ConfigMetadataProvider> configMetadataProvider = getConfigMetadataProvider();
170 if (configMetadataProvider != nullptr && configMetadataProvider->useV2SoftMemoryCalculation()) {
171 bool dimensionGuardrailHit = StatsdStats::getInstance().hasHitDimensionGuardrail(mMetricId);
172 return computeOverheadSizeLocked(!mPastBuckets.empty() || !mSkippedBuckets.empty(),
173 dimensionGuardrailHit) +
174 mTotalDataSize;
175 }
176 size_t totalSize = 0;
177 for (const auto& [_, buckets] : mPastBuckets) {
178 totalSize += buckets.size() * kBucketSize;
179 for (const auto& bucket : buckets) {
180 static const size_t kIntSize = sizeof(int);
181 totalSize += bucket.aggIndex.size() * kIntSize;
182 if (!bucket.aggregates.empty()) {
183 static const size_t kInt64Size = sizeof(int64_t);
184 // Assume sketch size is the same for all aggregations in a bucket.
185 totalSize += bucket.aggregates.size() * kInt64Size *
186 bucket.aggregates[0]->num_stored_values();
187 }
188 }
189 }
190 return totalSize;
191 }
192
193 } // namespace statsd
194 } // namespace os
195 } // namespace android
196