1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.systemui.screenrecord; 18 19 import static android.content.Context.MEDIA_PROJECTION_SERVICE; 20 21 import static com.android.systemui.screenrecord.ScreenRecordingAudioSource.INTERNAL; 22 import static com.android.systemui.screenrecord.ScreenRecordingAudioSource.MIC; 23 import static com.android.systemui.screenrecord.ScreenRecordingAudioSource.MIC_AND_INTERNAL; 24 25 import android.annotation.Nullable; 26 import android.app.ActivityManager; 27 import android.content.ContentResolver; 28 import android.content.ContentValues; 29 import android.content.Context; 30 import android.graphics.Bitmap; 31 import android.graphics.drawable.Icon; 32 import android.hardware.display.DisplayManager; 33 import android.hardware.display.VirtualDisplay; 34 import android.media.MediaCodec; 35 import android.media.MediaCodecInfo; 36 import android.media.MediaFormat; 37 import android.media.MediaMuxer; 38 import android.media.MediaRecorder; 39 import android.media.ThumbnailUtils; 40 import android.media.projection.IMediaProjection; 41 import android.media.projection.IMediaProjectionManager; 42 import android.media.projection.MediaProjection; 43 import android.media.projection.MediaProjectionManager; 44 import android.net.Uri; 45 import android.os.Handler; 46 import android.os.IBinder; 47 import android.os.RemoteException; 48 import android.os.ServiceManager; 49 import android.provider.MediaStore; 50 import android.util.DisplayMetrics; 51 import android.util.Log; 52 import android.util.Size; 53 import android.view.Surface; 54 import android.view.WindowManager; 55 56 import com.android.internal.R; 57 import com.android.systemui.mediaprojection.MediaProjectionCaptureTarget; 58 59 import java.io.Closeable; 60 import java.io.File; 61 import java.io.IOException; 62 import java.io.OutputStream; 63 import java.nio.file.Files; 64 import java.text.SimpleDateFormat; 65 import java.util.ArrayList; 66 import java.util.Date; 67 import java.util.List; 68 69 /** 70 * Recording screen and mic/internal audio 71 */ 72 public class ScreenMediaRecorder extends MediaProjection.Callback { 73 private static final int TOTAL_NUM_TRACKS = 1; 74 private static final int VIDEO_FRAME_RATE = 30; 75 private static final int VIDEO_FRAME_RATE_TO_RESOLUTION_RATIO = 6; 76 private static final int AUDIO_BIT_RATE = 196000; 77 private static final int AUDIO_SAMPLE_RATE = 44100; 78 private static final int MAX_DURATION_MS = 60 * 60 * 1000; 79 private static final long MAX_FILESIZE_BYTES = 5000000000L; 80 private static final String TAG = "ScreenMediaRecorder"; 81 82 83 private File mTempVideoFile; 84 private File mTempAudioFile; 85 private MediaProjection mMediaProjection; 86 private Surface mInputSurface; 87 private VirtualDisplay mVirtualDisplay; 88 private MediaRecorder mMediaRecorder; 89 private int mUid; 90 private ScreenRecordingMuxer mMuxer; 91 private ScreenInternalAudioRecorder mAudio; 92 private ScreenRecordingAudioSource mAudioSource; 93 private final MediaProjectionCaptureTarget mCaptureRegion; 94 private final Handler mHandler; 95 96 private Context mContext; 97 ScreenMediaRecorderListener mListener; 98 ScreenMediaRecorder(Context context, Handler handler, int uid, ScreenRecordingAudioSource audioSource, MediaProjectionCaptureTarget captureRegion, ScreenMediaRecorderListener listener)99 public ScreenMediaRecorder(Context context, Handler handler, 100 int uid, ScreenRecordingAudioSource audioSource, 101 MediaProjectionCaptureTarget captureRegion, 102 ScreenMediaRecorderListener listener) { 103 mContext = context; 104 mHandler = handler; 105 mUid = uid; 106 mCaptureRegion = captureRegion; 107 mListener = listener; 108 mAudioSource = audioSource; 109 } 110 prepare()111 private void prepare() throws IOException, RemoteException, RuntimeException { 112 //Setup media projection 113 IBinder b = ServiceManager.getService(MEDIA_PROJECTION_SERVICE); 114 IMediaProjectionManager mediaService = 115 IMediaProjectionManager.Stub.asInterface(b); 116 IMediaProjection proj = null; 117 proj = mediaService.createProjection(mUid, mContext.getPackageName(), 118 MediaProjectionManager.TYPE_SCREEN_CAPTURE, false); 119 IMediaProjection projection = IMediaProjection.Stub.asInterface(proj.asBinder()); 120 if (mCaptureRegion != null) { 121 projection.setLaunchCookie(mCaptureRegion.getLaunchCookie()); 122 projection.setTaskId(mCaptureRegion.getTaskId()); 123 } 124 mMediaProjection = new MediaProjection(mContext, projection); 125 mMediaProjection.registerCallback(this, mHandler); 126 127 File cacheDir = mContext.getCacheDir(); 128 cacheDir.mkdirs(); 129 mTempVideoFile = File.createTempFile("temp", ".mp4", cacheDir); 130 131 // Set up media recorder 132 mMediaRecorder = new MediaRecorder(); 133 134 // Set up audio source 135 if (mAudioSource == MIC) { 136 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.DEFAULT); 137 } 138 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 139 140 mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4); 141 142 143 // Set up video 144 DisplayMetrics metrics = new DisplayMetrics(); 145 WindowManager wm = (WindowManager) mContext.getSystemService(Context.WINDOW_SERVICE); 146 wm.getDefaultDisplay().getRealMetrics(metrics); 147 int refreshRate = (int) wm.getDefaultDisplay().getRefreshRate(); 148 int[] dimens = getSupportedSize(metrics.widthPixels, metrics.heightPixels, refreshRate); 149 int width = dimens[0]; 150 int height = dimens[1]; 151 refreshRate = dimens[2]; 152 int vidBitRate = width * height * refreshRate / VIDEO_FRAME_RATE 153 * VIDEO_FRAME_RATE_TO_RESOLUTION_RATIO; 154 mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264); 155 mMediaRecorder.setVideoEncodingProfileLevel( 156 MediaCodecInfo.CodecProfileLevel.AVCProfileHigh, 157 MediaCodecInfo.CodecProfileLevel.AVCLevel3); 158 mMediaRecorder.setVideoSize(width, height); 159 mMediaRecorder.setVideoFrameRate(refreshRate); 160 mMediaRecorder.setVideoEncodingBitRate(vidBitRate); 161 mMediaRecorder.setMaxDuration(MAX_DURATION_MS); 162 mMediaRecorder.setMaxFileSize(MAX_FILESIZE_BYTES); 163 164 // Set up audio 165 if (mAudioSource == MIC) { 166 mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.HE_AAC); 167 mMediaRecorder.setAudioChannels(TOTAL_NUM_TRACKS); 168 mMediaRecorder.setAudioEncodingBitRate(AUDIO_BIT_RATE); 169 mMediaRecorder.setAudioSamplingRate(AUDIO_SAMPLE_RATE); 170 } 171 172 mMediaRecorder.setOutputFile(mTempVideoFile); 173 mMediaRecorder.prepare(); 174 // Create surface 175 mInputSurface = mMediaRecorder.getSurface(); 176 mVirtualDisplay = mMediaProjection.createVirtualDisplay( 177 "Recording Display", 178 width, 179 height, 180 metrics.densityDpi, 181 DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, 182 mInputSurface, 183 new VirtualDisplay.Callback() { 184 @Override 185 public void onStopped() { 186 onStop(); 187 } 188 }, 189 mHandler); 190 191 mMediaRecorder.setOnInfoListener((mr, what, extra) -> mListener.onInfo(mr, what, extra)); 192 if (mAudioSource == INTERNAL || 193 mAudioSource == MIC_AND_INTERNAL) { 194 mTempAudioFile = File.createTempFile("temp", ".aac", 195 mContext.getCacheDir()); 196 mAudio = new ScreenInternalAudioRecorder(mTempAudioFile.getAbsolutePath(), 197 mMediaProjection, mAudioSource == MIC_AND_INTERNAL); 198 } 199 200 } 201 202 /** 203 * Find the highest supported screen resolution and refresh rate for the given dimensions on 204 * this device, up to actual size and given rate. 205 * If possible this will return the same values as given, but values may be smaller on some 206 * devices. 207 * 208 * @param screenWidth Actual pixel width of screen 209 * @param screenHeight Actual pixel height of screen 210 * @param refreshRate Desired refresh rate 211 * @return array with supported width, height, and refresh rate 212 */ getSupportedSize(final int screenWidth, final int screenHeight, int refreshRate)213 private int[] getSupportedSize(final int screenWidth, final int screenHeight, int refreshRate) 214 throws IOException { 215 String videoType = MediaFormat.MIMETYPE_VIDEO_AVC; 216 217 // Get max size from the decoder, to ensure recordings will be playable on device 218 MediaCodec decoder = MediaCodec.createDecoderByType(videoType); 219 MediaCodecInfo.VideoCapabilities vc = decoder.getCodecInfo() 220 .getCapabilitiesForType(videoType).getVideoCapabilities(); 221 decoder.release(); 222 223 // Check if we can support screen size as-is 224 int width = vc.getSupportedWidths().getUpper(); 225 int height = vc.getSupportedHeights().getUpper(); 226 227 int screenWidthAligned = screenWidth; 228 if (screenWidthAligned % vc.getWidthAlignment() != 0) { 229 screenWidthAligned -= (screenWidthAligned % vc.getWidthAlignment()); 230 } 231 int screenHeightAligned = screenHeight; 232 if (screenHeightAligned % vc.getHeightAlignment() != 0) { 233 screenHeightAligned -= (screenHeightAligned % vc.getHeightAlignment()); 234 } 235 236 if (width >= screenWidthAligned && height >= screenHeightAligned 237 && vc.isSizeSupported(screenWidthAligned, screenHeightAligned)) { 238 // Desired size is supported, now get the rate 239 int maxRate = vc.getSupportedFrameRatesFor(screenWidthAligned, 240 screenHeightAligned).getUpper().intValue(); 241 242 if (maxRate < refreshRate) { 243 refreshRate = maxRate; 244 } 245 Log.d(TAG, "Screen size supported at rate " + refreshRate); 246 return new int[]{screenWidthAligned, screenHeightAligned, refreshRate}; 247 } 248 249 // Otherwise, resize for max supported size 250 double scale = Math.min(((double) width / screenWidth), 251 ((double) height / screenHeight)); 252 253 int scaledWidth = (int) (screenWidth * scale); 254 int scaledHeight = (int) (screenHeight * scale); 255 if (scaledWidth % vc.getWidthAlignment() != 0) { 256 scaledWidth -= (scaledWidth % vc.getWidthAlignment()); 257 } 258 if (scaledHeight % vc.getHeightAlignment() != 0) { 259 scaledHeight -= (scaledHeight % vc.getHeightAlignment()); 260 } 261 262 // Find max supported rate for size 263 int maxRate = vc.getSupportedFrameRatesFor(scaledWidth, scaledHeight) 264 .getUpper().intValue(); 265 if (maxRate < refreshRate) { 266 refreshRate = maxRate; 267 } 268 269 Log.d(TAG, "Resized by " + scale + ": " + scaledWidth + ", " + scaledHeight 270 + ", " + refreshRate); 271 return new int[]{scaledWidth, scaledHeight, refreshRate}; 272 } 273 274 /** 275 * Start screen recording 276 */ start()277 void start() throws IOException, RemoteException, RuntimeException { 278 Log.d(TAG, "start recording"); 279 prepare(); 280 mMediaRecorder.start(); 281 recordInternalAudio(); 282 } 283 284 /** 285 * End screen recording, throws an exception if stopping recording failed 286 */ end()287 void end() throws IOException { 288 Closer closer = new Closer(); 289 290 // MediaRecorder might throw RuntimeException if stopped immediately after starting 291 // We should remove the recording in this case as it will be invalid 292 closer.register(mMediaRecorder::stop); 293 closer.register(mMediaRecorder::release); 294 closer.register(mInputSurface::release); 295 closer.register(mVirtualDisplay::release); 296 closer.register(mMediaProjection::stop); 297 closer.register(this::stopInternalAudioRecording); 298 299 closer.close(); 300 301 mMediaRecorder = null; 302 mMediaProjection = null; 303 304 Log.d(TAG, "end recording"); 305 } 306 307 @Override onStop()308 public void onStop() { 309 Log.d(TAG, "The system notified about stopping the projection"); 310 mListener.onStopped(); 311 } 312 stopInternalAudioRecording()313 private void stopInternalAudioRecording() { 314 if (mAudioSource == INTERNAL || mAudioSource == MIC_AND_INTERNAL) { 315 mAudio.end(); 316 mAudio = null; 317 } 318 } 319 recordInternalAudio()320 private void recordInternalAudio() throws IllegalStateException { 321 if (mAudioSource == INTERNAL || mAudioSource == MIC_AND_INTERNAL) { 322 mAudio.start(); 323 } 324 } 325 326 /** 327 * Store recorded video 328 */ save()329 protected SavedRecording save() throws IOException, IllegalStateException { 330 String fileName = new SimpleDateFormat("'screen-'yyyyMMdd-HHmmss'.mp4'") 331 .format(new Date()); 332 333 ContentValues values = new ContentValues(); 334 values.put(MediaStore.Video.Media.DISPLAY_NAME, fileName); 335 values.put(MediaStore.Video.Media.MIME_TYPE, "video/mp4"); 336 values.put(MediaStore.Video.Media.DATE_ADDED, System.currentTimeMillis()); 337 values.put(MediaStore.Video.Media.DATE_TAKEN, System.currentTimeMillis()); 338 339 ContentResolver resolver = mContext.getContentResolver(); 340 Uri collectionUri = MediaStore.Video.Media.getContentUri( 341 MediaStore.VOLUME_EXTERNAL_PRIMARY); 342 Uri itemUri = resolver.insert(collectionUri, values); 343 344 Log.d(TAG, itemUri.toString()); 345 if (mAudioSource == MIC_AND_INTERNAL || mAudioSource == INTERNAL) { 346 try { 347 Log.d(TAG, "muxing recording"); 348 File file = File.createTempFile("temp", ".mp4", 349 mContext.getCacheDir()); 350 mMuxer = new ScreenRecordingMuxer(MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4, 351 file.getAbsolutePath(), 352 mTempVideoFile.getAbsolutePath(), 353 mTempAudioFile.getAbsolutePath()); 354 mMuxer.mux(); 355 mTempVideoFile.delete(); 356 mTempVideoFile = file; 357 } catch (IOException e) { 358 Log.e(TAG, "muxing recording " + e.getMessage()); 359 e.printStackTrace(); 360 } 361 } 362 363 // Add to the mediastore 364 OutputStream os = resolver.openOutputStream(itemUri, "w"); 365 Files.copy(mTempVideoFile.toPath(), os); 366 os.close(); 367 if (mTempAudioFile != null) mTempAudioFile.delete(); 368 SavedRecording recording = new SavedRecording( 369 itemUri, mTempVideoFile, getRequiredThumbnailSize()); 370 mTempVideoFile.delete(); 371 return recording; 372 } 373 374 /** 375 * Returns the required {@code Size} of the thumbnail. 376 */ getRequiredThumbnailSize()377 private Size getRequiredThumbnailSize() { 378 boolean isLowRam = ActivityManager.isLowRamDeviceStatic(); 379 int thumbnailIconHeight = mContext.getResources().getDimensionPixelSize(isLowRam 380 ? R.dimen.notification_big_picture_max_height_low_ram 381 : R.dimen.notification_big_picture_max_height); 382 int thumbnailIconWidth = mContext.getResources().getDimensionPixelSize(isLowRam 383 ? R.dimen.notification_big_picture_max_width_low_ram 384 : R.dimen.notification_big_picture_max_width); 385 return new Size(thumbnailIconWidth, thumbnailIconHeight); 386 } 387 388 /** 389 * Release the resources without saving the data 390 */ release()391 protected void release() { 392 if (mTempVideoFile != null) { 393 mTempVideoFile.delete(); 394 } 395 if (mTempAudioFile != null) { 396 mTempAudioFile.delete(); 397 } 398 } 399 400 /** 401 * Object representing the recording 402 */ 403 public class SavedRecording { 404 405 private Uri mUri; 406 private Icon mThumbnailIcon; 407 SavedRecording(Uri uri, File file, Size thumbnailSize)408 protected SavedRecording(Uri uri, File file, Size thumbnailSize) { 409 mUri = uri; 410 try { 411 Bitmap thumbnailBitmap = ThumbnailUtils.createVideoThumbnail( 412 file, thumbnailSize, null); 413 mThumbnailIcon = Icon.createWithBitmap(thumbnailBitmap); 414 } catch (IOException e) { 415 Log.e(TAG, "Error creating thumbnail", e); 416 } 417 } 418 getUri()419 public Uri getUri() { 420 return mUri; 421 } 422 getThumbnail()423 public @Nullable Icon getThumbnail() { 424 return mThumbnailIcon; 425 } 426 } 427 428 interface ScreenMediaRecorderListener { 429 /** 430 * Called to indicate an info or a warning during recording. 431 * See {@link MediaRecorder.OnInfoListener} for the full description. 432 */ onInfo(MediaRecorder mr, int what, int extra)433 void onInfo(MediaRecorder mr, int what, int extra); 434 435 /** 436 * Called when the recording stopped by the system. 437 * For example, this might happen when doing partial screen sharing of an app 438 * and the app that is being captured is closed. 439 */ onStopped()440 void onStopped(); 441 } 442 443 /** 444 * Allows to register multiple {@link Closeable} objects and close them all by calling 445 * {@link Closer#close}. If there is an exception thrown during closing of one 446 * of the registered closeables it will continue trying closing the rest closeables. 447 * If there are one or more exceptions thrown they will be re-thrown at the end. 448 * In case of multiple exceptions only the first one will be thrown and all the rest 449 * will be printed. 450 */ 451 private static class Closer implements Closeable { 452 private final List<Closeable> mCloseables = new ArrayList<>(); 453 register(Closeable closeable)454 void register(Closeable closeable) { 455 mCloseables.add(closeable); 456 } 457 458 @Override close()459 public void close() throws IOException { 460 Throwable throwable = null; 461 462 for (int i = 0; i < mCloseables.size(); i++) { 463 Closeable closeable = mCloseables.get(i); 464 465 try { 466 closeable.close(); 467 } catch (Throwable e) { 468 if (throwable == null) { 469 throwable = e; 470 } else { 471 e.printStackTrace(); 472 } 473 } 474 } 475 476 if (throwable != null) { 477 if (throwable instanceof IOException) { 478 throw (IOException) throwable; 479 } 480 481 if (throwable instanceof RuntimeException) { 482 throw (RuntimeException) throwable; 483 } 484 485 throw (Error) throwable; 486 } 487 } 488 } 489 } 490