/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define LOG_TAG "MotionPredictor"
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
namespace input_flags = com::android::input::flags;
namespace android {
namespace {
/**
* Log debug messages about predictions.
* Enable this via "adb shell setprop log.tag.MotionPredictor DEBUG"
*/
bool isDebug() {
return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG, ANDROID_LOG_INFO);
}
// Converts a prediction of some polar (r, phi) to Cartesian (x, y) when applied to an axis.
TfLiteMotionPredictorSample::Point convertPrediction(
const TfLiteMotionPredictorSample::Point& axisFrom,
const TfLiteMotionPredictorSample::Point& axisTo, float r, float phi) {
const TfLiteMotionPredictorSample::Point axis = axisTo - axisFrom;
const float axis_phi = std::atan2(axis.y, axis.x);
const float x_delta = r * std::cos(axis_phi + phi);
const float y_delta = r * std::sin(axis_phi + phi);
return {.x = axisTo.x + x_delta, .y = axisTo.y + y_delta};
}
float normalizeRange(float x, float min, float max) {
const float normalized = (x - min) / (max - min);
return std::min(1.0f, std::max(0.0f, normalized));
}
} // namespace
// --- JerkTracker ---
JerkTracker::JerkTracker(bool normalizedDt) : mNormalizedDt(normalizedDt) {}
void JerkTracker::pushSample(int64_t timestamp, float xPos, float yPos) {
mTimestamps.pushBack(timestamp);
const int numSamples = mTimestamps.size();
std::array newXDerivatives;
std::array newYDerivatives;
/**
* Diagram showing the calculation of higher order derivatives of sample x3
* collected at time=t3.
* Terms in parentheses are not stored (and not needed for calculations)
* t0 ----- t1 ----- t2 ----- t3
* (x0)-----(x1) ----- x2 ----- x3
* (x'0) --- x'1 --- x'2
* x''0 - x''1
* x'''0
*
* In this example:
* x'2 = (x3 - x2) / (t3 - t2)
* x''1 = (x'2 - x'1) / (t2 - t1)
* x'''0 = (x''1 - x''0) / (t1 - t0)
* Therefore, timestamp history is needed to calculate higher order derivatives,
* compared to just the last calculated derivative sample.
*
* If mNormalizedDt = true, then dt = 1 and the division is moot.
*/
for (int i = 0; i < numSamples; ++i) {
if (i == 0) {
newXDerivatives[i] = xPos;
newYDerivatives[i] = yPos;
} else {
newXDerivatives[i] = newXDerivatives[i - 1] - mXDerivatives[i - 1];
newYDerivatives[i] = newYDerivatives[i - 1] - mYDerivatives[i - 1];
if (!mNormalizedDt) {
const float dt = mTimestamps[numSamples - i] - mTimestamps[numSamples - i - 1];
newXDerivatives[i] = newXDerivatives[i] / dt;
newYDerivatives[i] = newYDerivatives[i] / dt;
}
}
}
std::swap(newXDerivatives, mXDerivatives);
std::swap(newYDerivatives, mYDerivatives);
}
void JerkTracker::reset() {
mTimestamps.clear();
}
std::optional JerkTracker::jerkMagnitude() const {
if (mTimestamps.size() == mTimestamps.capacity()) {
return std::hypot(mXDerivatives[3], mYDerivatives[3]);
}
return std::nullopt;
}
// --- MotionPredictor ---
MotionPredictor::MotionPredictor(nsecs_t predictionTimestampOffsetNanos,
std::function checkMotionPredictionEnabled,
ReportAtomFunction reportAtomFunction)
: mPredictionTimestampOffsetNanos(predictionTimestampOffsetNanos),
mCheckMotionPredictionEnabled(std::move(checkMotionPredictionEnabled)),
mReportAtomFunction(reportAtomFunction) {}
android::base::Result MotionPredictor::record(const MotionEvent& event) {
if (mLastEvent && mLastEvent->getDeviceId() != event.getDeviceId()) {
// We still have an active gesture for another device. The provided MotionEvent is not
// consistent with the previous gesture.
LOG(ERROR) << "Inconsistent event stream: last event is " << *mLastEvent << ", but "
<< __func__ << " is called with " << event;
return android::base::Error()
<< "Inconsistent event stream: still have an active gesture from device "
<< mLastEvent->getDeviceId() << ", but received " << event;
}
if (!isPredictionAvailable(event.getDeviceId(), event.getSource())) {
ALOGE("Prediction not supported for device %d's %s source", event.getDeviceId(),
inputEventSourceToString(event.getSource()).c_str());
return {};
}
// Initialise the model now that it's likely to be used.
if (!mModel) {
mModel = TfLiteMotionPredictorModel::create();
LOG_ALWAYS_FATAL_IF(!mModel);
}
if (!mBuffers) {
mBuffers = std::make_unique(mModel->inputLength());
}
// Pass input event to the MetricsManager.
if (!mMetricsManager) {
mMetricsManager.emplace(mModel->config().predictionInterval, mModel->outputLength(),
mReportAtomFunction);
}
mMetricsManager->onRecord(event);
const int32_t action = event.getActionMasked();
if (action == AMOTION_EVENT_ACTION_UP || action == AMOTION_EVENT_ACTION_CANCEL) {
ALOGD_IF(isDebug(), "End of event stream");
mBuffers->reset();
mJerkTracker.reset();
mLastEvent.reset();
return {};
} else if (action != AMOTION_EVENT_ACTION_DOWN && action != AMOTION_EVENT_ACTION_MOVE) {
ALOGD_IF(isDebug(), "Skipping unsupported %s action",
MotionEvent::actionToString(action).c_str());
return {};
}
if (event.getPointerCount() != 1) {
ALOGD_IF(isDebug(), "Prediction not supported for multiple pointers");
return {};
}
const ToolType toolType = event.getPointerProperties(0)->toolType;
if (toolType != ToolType::STYLUS) {
ALOGD_IF(isDebug(), "Prediction not supported for non-stylus tool: %s",
ftl::enum_string(toolType).c_str());
return {};
}
for (size_t i = 0; i <= event.getHistorySize(); ++i) {
if (event.isResampled(0, i)) {
continue;
}
const PointerCoords* coords = event.getHistoricalRawPointerCoords(0, i);
mBuffers->pushSample(event.getHistoricalEventTime(i),
{
.position.x = coords->getAxisValue(AMOTION_EVENT_AXIS_X),
.position.y = coords->getAxisValue(AMOTION_EVENT_AXIS_Y),
.pressure = event.getHistoricalPressure(0, i),
.tilt = event.getHistoricalAxisValue(AMOTION_EVENT_AXIS_TILT,
0, i),
.orientation = event.getHistoricalOrientation(0, i),
});
mJerkTracker.pushSample(event.getHistoricalEventTime(i),
coords->getAxisValue(AMOTION_EVENT_AXIS_X),
coords->getAxisValue(AMOTION_EVENT_AXIS_Y));
}
if (!mLastEvent) {
mLastEvent = MotionEvent();
}
mLastEvent->copyFrom(&event, /*keepHistory=*/false);
return {};
}
std::unique_ptr MotionPredictor::predict(nsecs_t timestamp) {
if (mBuffers == nullptr || !mBuffers->isReady()) {
return nullptr;
}
LOG_ALWAYS_FATAL_IF(!mModel);
mBuffers->copyTo(*mModel);
LOG_ALWAYS_FATAL_IF(!mModel->invoke());
// Read out the predictions.
const std::span predictedR = mModel->outputR();
const std::span predictedPhi = mModel->outputPhi();
const std::span predictedPressure = mModel->outputPressure();
TfLiteMotionPredictorSample::Point axisFrom = mBuffers->axisFrom().position;
TfLiteMotionPredictorSample::Point axisTo = mBuffers->axisTo().position;
if (isDebug()) {
ALOGD("axisFrom: %f, %f", axisFrom.x, axisFrom.y);
ALOGD("axisTo: %f, %f", axisTo.x, axisTo.y);
ALOGD("mInputR: %s", base::Join(mModel->inputR(), ", ").c_str());
ALOGD("mInputPhi: %s", base::Join(mModel->inputPhi(), ", ").c_str());
ALOGD("mInputPressure: %s", base::Join(mModel->inputPressure(), ", ").c_str());
ALOGD("mInputTilt: %s", base::Join(mModel->inputTilt(), ", ").c_str());
ALOGD("mInputOrientation: %s", base::Join(mModel->inputOrientation(), ", ").c_str());
ALOGD("predictedR: %s", base::Join(predictedR, ", ").c_str());
ALOGD("predictedPhi: %s", base::Join(predictedPhi, ", ").c_str());
ALOGD("predictedPressure: %s", base::Join(predictedPressure, ", ").c_str());
}
LOG_ALWAYS_FATAL_IF(!mLastEvent);
const MotionEvent& event = *mLastEvent;
bool hasPredictions = false;
std::unique_ptr prediction = std::make_unique();
int64_t predictionTime = mBuffers->lastTimestamp();
const int64_t futureTime = timestamp + mPredictionTimestampOffsetNanos;
const float jerkMagnitude = mJerkTracker.jerkMagnitude().value_or(0);
const float fractionKept =
1 - normalizeRange(jerkMagnitude, mModel->config().lowJerk, mModel->config().highJerk);
// float to ensure proper division below.
const float predictionTimeWindow = futureTime - predictionTime;
const int maxNumPredictions = static_cast(
std::ceil(predictionTimeWindow / mModel->config().predictionInterval * fractionKept));
ALOGD_IF(isDebug(),
"jerk (d^3p/normalizedDt^3): %f, fraction of prediction window pruned: %f, max number "
"of predictions: %d",
jerkMagnitude, 1 - fractionKept, maxNumPredictions);
for (size_t i = 0; i < static_cast(predictedR.size()) && predictionTime <= futureTime;
++i) {
if (predictedR[i] < mModel->config().distanceNoiseFloor) {
// Stop predicting when the predicted output is below the model's noise floor.
//
// We assume that all subsequent predictions in the batch are unreliable because later
// predictions are conditional on earlier predictions, and a state of noise is not a
// good basis for prediction.
//
// The UX trade-off is that this potentially sacrifices some predictions when the input
// device starts to speed up, but avoids producing noisy predictions as it slows down.
break;
}
if (input_flags::enable_prediction_pruning_via_jerk_thresholding()) {
if (i >= static_cast(maxNumPredictions)) {
break;
}
}
// TODO(b/266747654): Stop predictions if confidence is < some
// threshold. Currently predictions are pruned via jerk thresholding.
const TfLiteMotionPredictorSample::Point predictedPoint =
convertPrediction(axisFrom, axisTo, predictedR[i], predictedPhi[i]);
ALOGD_IF(isDebug(), "prediction %zu: %f, %f", i, predictedPoint.x, predictedPoint.y);
PointerCoords coords;
coords.clear();
coords.setAxisValue(AMOTION_EVENT_AXIS_X, predictedPoint.x);
coords.setAxisValue(AMOTION_EVENT_AXIS_Y, predictedPoint.y);
coords.setAxisValue(AMOTION_EVENT_AXIS_PRESSURE, predictedPressure[i]);
// Copy forward tilt and orientation from the last event until they are predicted
// (b/291789258).
coords.setAxisValue(AMOTION_EVENT_AXIS_TILT,
event.getAxisValue(AMOTION_EVENT_AXIS_TILT, 0));
coords.setAxisValue(AMOTION_EVENT_AXIS_ORIENTATION,
event.getRawPointerCoords(0)->getAxisValue(
AMOTION_EVENT_AXIS_ORIENTATION));
predictionTime += mModel->config().predictionInterval;
if (i == 0) {
hasPredictions = true;
prediction->initialize(InputEvent::nextId(), event.getDeviceId(), event.getSource(),
event.getDisplayId(), INVALID_HMAC, AMOTION_EVENT_ACTION_MOVE,
event.getActionButton(), event.getFlags(), event.getEdgeFlags(),
event.getMetaState(), event.getButtonState(),
event.getClassification(), event.getTransform(),
event.getXPrecision(), event.getYPrecision(),
event.getRawXCursorPosition(), event.getRawYCursorPosition(),
event.getRawTransform(), event.getDownTime(), predictionTime,
event.getPointerCount(), event.getPointerProperties(), &coords);
} else {
prediction->addSample(predictionTime, &coords);
}
axisFrom = axisTo;
axisTo = predictedPoint;
}
if (!hasPredictions) {
return nullptr;
}
// Pass predictions to the MetricsManager.
LOG_ALWAYS_FATAL_IF(!mMetricsManager);
mMetricsManager->onPredict(*prediction);
return prediction;
}
bool MotionPredictor::isPredictionAvailable(int32_t /*deviceId*/, int32_t source) {
// Global flag override
if (!mCheckMotionPredictionEnabled()) {
ALOGD_IF(isDebug(), "Prediction not available due to flag override");
return false;
}
// Prediction is only supported for stylus sources.
if (!isFromSource(source, AINPUT_SOURCE_STYLUS)) {
ALOGD_IF(isDebug(), "Prediction not available for non-stylus source: %s",
inputEventSourceToString(source).c_str());
return false;
}
return true;
}
} // namespace android