1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.google.android.test.handwritingime;
17 
18 import android.R;
19 import android.annotation.Nullable;
20 import android.graphics.PointF;
21 import android.graphics.RectF;
22 import android.inputmethodservice.InputMethodService;
23 import android.os.CancellationSignal;
24 import android.os.Handler;
25 import android.util.Log;
26 import android.view.MotionEvent;
27 import android.view.View;
28 import android.view.ViewGroup;
29 import android.view.Window;
30 import android.view.inputmethod.CursorAnchorInfo;
31 import android.view.inputmethod.DeleteGesture;
32 import android.view.inputmethod.EditorInfo;
33 import android.view.inputmethod.HandwritingGesture;
34 import android.view.inputmethod.InputConnection;
35 import android.view.inputmethod.InsertGesture;
36 import android.view.inputmethod.InsertModeGesture;
37 import android.view.inputmethod.JoinOrSplitGesture;
38 import android.view.inputmethod.PreviewableHandwritingGesture;
39 import android.view.inputmethod.RemoveSpaceGesture;
40 import android.view.inputmethod.SelectGesture;
41 import android.widget.AdapterView;
42 import android.widget.ArrayAdapter;
43 import android.widget.CheckBox;
44 import android.widget.FrameLayout;
45 import android.widget.LinearLayout;
46 import android.widget.Spinner;
47 import android.widget.Toast;
48 
49 import java.util.Random;
50 import java.util.function.IntConsumer;
51 
52 public class HandwritingIme extends InputMethodService {
53     private static final int OP_NONE = 0;
54     // ------- PreviewableHandwritingGesture BEGIN -----
55     private static final int OP_SELECT = 1;
56     private static final int OP_DELETE = 2;
57     // ------- PreviewableHandwritingGesture END -----
58     private static final int OP_INSERT = 3;
59     private static final int OP_REMOVE_SPACE = 4;
60     private static final int OP_JOIN_OR_SPLIT = 5;
61     private static final int OP_INSERT_MODE = 6;
62 
63     private InkView mInk;
64 
65     static final String TAG = "HandwritingIme";
66     private int mRichGestureMode = OP_NONE;
67     private int mRichGestureGranularity = -1;
68     private Spinner mRichGestureModeSpinner;
69     private Spinner mRichGestureGranularitySpinner;
70     private PointF mRichGestureStartPoint;
71 
72     static final int BOUNDS_INFO_NONE = 0;
73     static final int BOUNDS_INFO_VISIBLE_LINE_BOUNDS = 1;
74     static final int BOUNDS_INFO_EDITOR_BOUNDS = 2;
75     private int mBoundsInfoMode = BOUNDS_INFO_NONE;
76     private LinearLayout mBoundsInfoCheckBoxes;
77 
78     private final IntConsumer mResultConsumer = value -> Log.d(TAG, "Gesture result: " + value);
79 
80     private CancellationSignal mCancellationSignal = new CancellationSignal();
81     private boolean mUsePreview;
82     private CheckBox mGesturePreviewCheckbox;
83 
84     interface HandwritingFinisher {
finish()85         void finish();
86     }
87 
88     interface StylusListener {
onStylusEvent(MotionEvent me)89         void onStylusEvent(MotionEvent me);
90     }
91 
92     final class StylusConsumer implements StylusListener {
93         @Override
onStylusEvent(MotionEvent me)94         public void onStylusEvent(MotionEvent me) {
95             HandwritingIme.this.onStylusEvent(me);
96         }
97     }
98 
99     final class HandwritingFinisherImpl implements HandwritingFinisher {
100 
HandwritingFinisherImpl()101         HandwritingFinisherImpl() {}
102 
103         @Override
finish()104         public void finish() {
105             finishStylusHandwriting();
106             Log.d(TAG, "HandwritingIme called finishStylusHandwriting() ");
107         }
108     }
109 
onStylusEvent(@ullable MotionEvent event)110     private void onStylusEvent(@Nullable MotionEvent event) {
111         // TODO Hookup recognizer here
112         HandwritingGesture gesture;
113         switch (event.getAction()) {
114             case MotionEvent.ACTION_MOVE:
115                 if (mUsePreview && areRichGesturesEnabled()) {
116                     gesture = computeGesture(event, true /* isPreview */);
117                     if (gesture == null) {
118                         Log.e(TAG, "Preview not supported for gesture: " + mRichGestureMode);
119                         return;
120                     }
121                     performGesture(gesture, true /* isPreview */);
122                 }
123                 break;
124             case MotionEvent.ACTION_UP:
125                 if (areRichGesturesEnabled()) {
126                     gesture = computeGesture(event, false /* isPreview */);
127                     if (gesture == null) {
128                         // This shouldn't happen
129                         Log.e(TAG, "Unrecognized gesture mode: " + mRichGestureMode);
130                         return;
131                     }
132                     performGesture(gesture, false /* isPreview */);
133                 } else {
134                     // insert random ASCII char
135                     sendKeyChar((char) (56 + new Random().nextInt(66)));
136                 }
137                 return;
138             case MotionEvent.ACTION_DOWN: {
139                 if (areRichGesturesEnabled()) {
140                     mRichGestureStartPoint = new PointF(event.getX(), event.getY());
141                 }
142             }
143         }
144     }
145 
computeGesture(MotionEvent event, boolean isPreview)146     private HandwritingGesture computeGesture(MotionEvent event, boolean isPreview) {
147         HandwritingGesture gesture = null;
148         switch (mRichGestureMode) {
149             case OP_SELECT:
150                 gesture = new SelectGesture.Builder()
151                         .setGranularity(mRichGestureGranularity)
152                         .setSelectionArea(getSanitizedRectF(mRichGestureStartPoint.x,
153                                 mRichGestureStartPoint.y, event.getX(), event.getY()))
154                         .setFallbackText("fallback text")
155                         .build();
156                 break;
157             case OP_DELETE:
158                 gesture = new DeleteGesture.Builder()
159                         .setGranularity(mRichGestureGranularity)
160                         .setDeletionArea(getSanitizedRectF(mRichGestureStartPoint.x,
161                                 mRichGestureStartPoint.y, event.getX(), event.getY()))
162                         .setFallbackText("fallback text")
163                         .build();
164                 break;
165             case OP_INSERT:
166                 gesture = new InsertGesture.Builder()
167                         .setInsertionPoint(new PointF(
168                                 mRichGestureStartPoint.x, mRichGestureStartPoint.y))
169                         .setTextToInsert(" ")
170                         .setFallbackText("fallback text")
171                         .build();
172                 break;
173             case OP_REMOVE_SPACE:
174                 if (isPreview) {
175                     break;
176                 }
177                 gesture = new RemoveSpaceGesture.Builder()
178                         .setPoints(
179                                 new PointF(mRichGestureStartPoint.x,
180                                         mRichGestureStartPoint.y),
181                                 new PointF(event.getX(), event.getY()))
182                         .setFallbackText("fallback text")
183                         .build();
184                 break;
185             case OP_JOIN_OR_SPLIT:
186                 if (isPreview) {
187                     break;
188                 }
189                 gesture = new JoinOrSplitGesture.Builder()
190                         .setJoinOrSplitPoint(new PointF(
191                                 mRichGestureStartPoint.x, mRichGestureStartPoint.y))
192                         .setFallbackText("fallback text")
193                         .build();
194                 break;
195             case OP_INSERT_MODE:
196                 if (isPreview) {
197                     break;
198                 }
199                 mCancellationSignal = new CancellationSignal();
200                 InsertModeGesture img = new InsertModeGesture.Builder()
201                         .setInsertionPoint(new PointF(
202                                 mRichGestureStartPoint.x, mRichGestureStartPoint.y))
203                         .setFallbackText("fallback text")
204                         .setCancellationSignal(mCancellationSignal)
205                         .build();
206                 gesture = img;
207                 new Handler().postDelayed(() -> img.getCancellationSignal().cancel(), 5000);
208                 break;
209         }
210         return gesture;
211     }
212 
213     /**
214      * sanitize values to support rectangles in all cases.
215      */
getSanitizedRectF(float left, float top, float right, float bottom)216     private RectF getSanitizedRectF(float left, float top, float right, float bottom) {
217         // swap values when left > right OR top > bottom.
218         if (left > right) {
219             float temp = left;
220             left = right;
221             right = temp;
222         }
223         if (top > bottom) {
224             float temp = top;
225             top = bottom;
226             bottom = temp;
227         }
228         // increment by a pixel so that RectF.isEmpty() isn't true.
229         if (left == right) {
230             right++;
231         }
232         if (top == bottom) {
233             bottom++;
234         }
235 
236         RectF rectF = new RectF(left, top, right, bottom);
237         Log.d(TAG, "Sending RichGesture " + rectF.toShortString());
238         return rectF;
239     }
240 
performGesture(HandwritingGesture gesture, boolean isPreview)241     private void performGesture(HandwritingGesture gesture, boolean isPreview) {
242         InputConnection ic = getCurrentInputConnection();
243         if (getCurrentInputStarted() && ic != null) {
244             if (isPreview) {
245                 ic.previewHandwritingGesture((PreviewableHandwritingGesture) gesture, null);
246             } else {
247                 ic.performHandwritingGesture(gesture, Runnable::run, mResultConsumer);
248             }
249         } else {
250             // This shouldn't happen
251             Log.e(TAG, "No active InputConnection");
252         }
253     }
254 
255     @Override
onCreateInputView()256     public View onCreateInputView() {
257         Log.d(TAG, "onCreateInputView");
258         final ViewGroup view = new FrameLayout(this);
259         view.setPadding(0, 0, 0, 0);
260 
261         LinearLayout layout = new LinearLayout(this);
262         layout.setLayoutParams(new LinearLayout.LayoutParams(
263                 ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT));
264         layout.setOrientation(LinearLayout.VERTICAL);
265         layout.addView(getRichGestureActionsSpinner());
266         layout.addView(getRichGestureGranularitySpinner());
267         layout.addView(getBoundsInfoCheckBoxes());
268         layout.addView(getPreviewCheckBox());
269         layout.setBackgroundColor(getColor(R.color.holo_green_light));
270         view.addView(layout);
271 
272         return view;
273     }
274 
getPreviewCheckBox()275     private View getPreviewCheckBox() {
276         mGesturePreviewCheckbox = new CheckBox(this);
277         mGesturePreviewCheckbox.setText("Use Gesture Previews (for Previewable Gestures)");
278         mGesturePreviewCheckbox.setOnCheckedChangeListener(
279                 (buttonView, isChecked) -> mUsePreview = isChecked);
280         return mGesturePreviewCheckbox;
281     }
282 
getRichGestureActionsSpinner()283     private View getRichGestureActionsSpinner() {
284         if (mRichGestureModeSpinner != null) {
285             return mRichGestureModeSpinner;
286         }
287         mRichGestureModeSpinner = new Spinner(this);
288         mRichGestureModeSpinner.setPadding(100, 0, 100, 0);
289         mRichGestureModeSpinner.setTooltipText("Handwriting IME mode");
290         String[] items = new String[]{
291                 "Handwriting IME - Rich gesture disabled",
292                 "Rich gesture SELECT",
293                 "Rich gesture DELETE",
294                 "Rich gesture INSERT",
295                 "Rich gesture REMOVE SPACE",
296                 "Rich gesture JOIN OR SPLIT",
297                 "Rich gesture INSERT MODE",
298         };
299         ArrayAdapter<String> adapter = new ArrayAdapter<>(this,
300                 android.R.layout.simple_spinner_dropdown_item, items);
301         adapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item);
302         mRichGestureModeSpinner.setAdapter(adapter);
303         mRichGestureModeSpinner.setOnItemSelectedListener(new AdapterView.OnItemSelectedListener() {
304             @Override
305             public void onItemSelected(AdapterView<?> parent, View view, int position, long id) {
306                 mRichGestureMode = position;
307                 boolean supportsGranularityAndPreview =
308                         mRichGestureMode == OP_SELECT || mRichGestureMode == OP_DELETE;
309                 mRichGestureGranularitySpinner.setEnabled(supportsGranularityAndPreview);
310                 mGesturePreviewCheckbox.setEnabled(supportsGranularityAndPreview);
311                 if (!supportsGranularityAndPreview) {
312                     mUsePreview = false;
313                 }
314                 Log.d(TAG, "Setting RichGesture Mode " + mRichGestureMode);
315             }
316 
317             @Override
318             public void onNothingSelected(AdapterView<?> parent) {
319                 mRichGestureMode = OP_NONE;
320                 mRichGestureGranularitySpinner.setEnabled(false);
321             }
322         });
323         mRichGestureModeSpinner.setSelection(0); // default disabled
324         return mRichGestureModeSpinner;
325     }
326 
updateCursorAnchorInfo(int boundsInfoMode)327     private void updateCursorAnchorInfo(int boundsInfoMode) {
328         final InputConnection ic = getCurrentInputConnection();
329         if (ic == null) return;
330 
331         if (boundsInfoMode == BOUNDS_INFO_NONE) {
332             ic.requestCursorUpdates(0);
333             return;
334         }
335 
336         final int cursorUpdateMode = InputConnection.CURSOR_UPDATE_MONITOR;
337         int cursorUpdateFilter = 0;
338         if ((boundsInfoMode & BOUNDS_INFO_EDITOR_BOUNDS) != 0) {
339             cursorUpdateFilter |= InputConnection.CURSOR_UPDATE_FILTER_EDITOR_BOUNDS;
340         }
341 
342         if ((boundsInfoMode & BOUNDS_INFO_VISIBLE_LINE_BOUNDS) != 0) {
343             cursorUpdateFilter |= InputConnection.CURSOR_UPDATE_FILTER_VISIBLE_LINE_BOUNDS;
344         }
345         ic.requestCursorUpdates(cursorUpdateMode | cursorUpdateFilter);
346     }
347 
updateBoundsInfoMode()348     private void updateBoundsInfoMode() {
349         if (mInk != null) {
350             mInk.setBoundsInfoMode(mBoundsInfoMode);
351         }
352         updateCursorAnchorInfo(mBoundsInfoMode);
353     }
354 
getBoundsInfoCheckBoxes()355     private View getBoundsInfoCheckBoxes() {
356         if (mBoundsInfoCheckBoxes != null) {
357             return mBoundsInfoCheckBoxes;
358         }
359         mBoundsInfoCheckBoxes = new LinearLayout(this);
360         mBoundsInfoCheckBoxes.setPadding(100, 0, 100, 0);
361         mBoundsInfoCheckBoxes.setOrientation(LinearLayout.HORIZONTAL);
362 
363         final CheckBox editorBoundsInfoCheckBox = new CheckBox(this);
364         editorBoundsInfoCheckBox.setText("EditorBoundsInfo");
365         editorBoundsInfoCheckBox.setOnCheckedChangeListener((buttonView, isChecked) -> {
366             if (isChecked) {
367                 mBoundsInfoMode |= BOUNDS_INFO_EDITOR_BOUNDS;
368             } else {
369                 mBoundsInfoMode &= ~BOUNDS_INFO_EDITOR_BOUNDS;
370             }
371             updateBoundsInfoMode();
372         });
373 
374         final CheckBox visibleLineBoundsInfoCheckBox = new CheckBox(this);
375         visibleLineBoundsInfoCheckBox.setText("VisibleLineBounds");
376         visibleLineBoundsInfoCheckBox.setOnCheckedChangeListener((buttonView, isChecked) -> {
377             if (isChecked) {
378                 mBoundsInfoMode |= BOUNDS_INFO_VISIBLE_LINE_BOUNDS;
379             } else {
380                 mBoundsInfoMode &= ~BOUNDS_INFO_VISIBLE_LINE_BOUNDS;
381             }
382             updateBoundsInfoMode();
383         });
384 
385         mBoundsInfoCheckBoxes.addView(editorBoundsInfoCheckBox);
386         mBoundsInfoCheckBoxes.addView(visibleLineBoundsInfoCheckBox);
387         return mBoundsInfoCheckBoxes;
388     }
389 
getRichGestureGranularitySpinner()390     private View getRichGestureGranularitySpinner() {
391         if (mRichGestureGranularitySpinner != null) {
392             return mRichGestureGranularitySpinner;
393         }
394         mRichGestureGranularitySpinner = new Spinner(this);
395         mRichGestureGranularitySpinner.setPadding(100, 0, 100, 0);
396         mRichGestureGranularitySpinner.setTooltipText(" Granularity");
397         String[] items =
398                 new String[] { "Granularity - UNDEFINED",
399                         "Granularity - WORD", "Granularity - CHARACTER"};
400         ArrayAdapter<String> adapter = new ArrayAdapter<>(this,
401                 android.R.layout.simple_spinner_dropdown_item, items);
402         adapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item);
403         mRichGestureGranularitySpinner.setAdapter(adapter);
404         mRichGestureGranularitySpinner.setOnItemSelectedListener(
405                 new AdapterView.OnItemSelectedListener() {
406             @Override
407             public void onItemSelected(AdapterView<?> parent, View view, int position, long id) {
408                 mRichGestureGranularity = position;
409                 Log.d(TAG, "Setting RichGesture Granularity " + mRichGestureGranularity);
410             }
411 
412             @Override
413             public void onNothingSelected(AdapterView<?> parent) {
414                 mRichGestureGranularity = 0;
415             }
416         });
417         mRichGestureGranularitySpinner.setSelection(1);
418         return mRichGestureGranularitySpinner;
419     }
420 
onPrepareStylusHandwriting()421     public void onPrepareStylusHandwriting() {
422         Log.d(TAG, "onPrepareStylusHandwriting ");
423         if (mInk == null) {
424             mInk = new InkView(this, new HandwritingFinisherImpl(), new StylusConsumer());
425             mInk.setBoundsInfoMode(mBoundsInfoMode);
426         }
427     }
428 
429     @Override
onStartStylusHandwriting()430     public boolean onStartStylusHandwriting() {
431         Log.d(TAG, "onStartStylusHandwriting ");
432         Toast.makeText(this, "START HW", Toast.LENGTH_SHORT).show();
433         Window inkWindow = getStylusHandwritingWindow();
434         inkWindow.setContentView(mInk, mInk.getLayoutParams());
435         return true;
436     }
437 
438     @Override
onFinishStylusHandwriting()439     public void onFinishStylusHandwriting() {
440         Log.d(TAG, "onFinishStylusHandwriting ");
441         Toast.makeText(this, "Finish HW", Toast.LENGTH_SHORT).show();
442         // Free-up
443         ((ViewGroup) mInk.getParent()).removeView(mInk);
444         mInk = null;
445     }
446 
447     @Override
onEvaluateFullscreenMode()448     public boolean onEvaluateFullscreenMode() {
449         return false;
450     }
451 
areRichGesturesEnabled()452     private boolean areRichGesturesEnabled() {
453         return mRichGestureMode != OP_NONE;
454     }
455 
456     @Override
onUpdateCursorAnchorInfo(CursorAnchorInfo cursorAnchorInfo)457     public void onUpdateCursorAnchorInfo(CursorAnchorInfo cursorAnchorInfo) {
458         if (mInk != null) {
459             mInk.setCursorAnchorInfo(cursorAnchorInfo);
460         }
461     }
462 
463     @Override
onStartInput(EditorInfo attribute, boolean restarting)464     public void onStartInput(EditorInfo attribute, boolean restarting) {
465         updateCursorAnchorInfo(mBoundsInfoMode);
466     }
467 }
468