1 /*crosvm
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "host/libs/vm_manager/crosvm_manager.h"
18
19 #include <poll.h>
20 #include <signal.h>
21 #include <sys/stat.h>
22 #include <sys/types.h>
23
24 #include <cassert>
25 #include <string>
26 #include <unordered_map>
27 #include <utility>
28 #include <vector>
29
30 #include <android-base/file.h>
31 #include <android-base/logging.h>
32 #include <android-base/strings.h>
33 #include <json/json.h>
34 #include <vulkan/vulkan.h>
35
36 #include "common/libs/utils/environment.h"
37 #include "common/libs/utils/files.h"
38 #include "common/libs/utils/json.h"
39 #include "common/libs/utils/network.h"
40 #include "common/libs/utils/result.h"
41 #include "common/libs/utils/subprocess.h"
42 #include "host/libs/command_util/snapshot_utils.h"
43 #include "host/libs/config/cuttlefish_config.h"
44 #include "host/libs/config/known_paths.h"
45 #include "host/libs/vm_manager/crosvm_builder.h"
46 #include "host/libs/vm_manager/qemu_manager.h"
47
48 namespace cuttlefish {
49 namespace vm_manager {
50
51 constexpr auto kTouchpadDefaultPrefix = "Crosvm_Virtio_Multitouch_Touchpad_";
52
IsSupported()53 bool CrosvmManager::IsSupported() {
54 #ifdef __ANDROID__
55 return true;
56 #else
57 return HostSupportsQemuCli();
58 #endif
59 }
60
61 Result<std::unordered_map<std::string, std::string>>
ConfigureGraphics(const CuttlefishConfig::InstanceSpecific & instance)62 CrosvmManager::ConfigureGraphics(
63 const CuttlefishConfig::InstanceSpecific& instance) {
64 // Override the default HAL search paths in all cases. We do this because
65 // the HAL search path allows for fallbacks, and fallbacks in conjunction
66 // with properties lead to non-deterministic behavior while loading the
67 // HALs.
68
69 std::unordered_map<std::string, std::string> bootconfig_args;
70
71 if (instance.gpu_mode() == kGpuModeGuestSwiftshader) {
72 bootconfig_args = {
73 {"androidboot.cpuvulkan.version", std::to_string(VK_API_VERSION_1_2)},
74 {"androidboot.hardware.gralloc", "minigbm"},
75 {"androidboot.hardware.hwcomposer", instance.hwcomposer()},
76 {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
77 {"androidboot.hardware.hwcomposer.display_framebuffer_format",
78 instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
79 {"androidboot.hardware.egl", "angle"},
80 {"androidboot.hardware.vulkan", "pastel"},
81 {"androidboot.opengles.version", "196609"}, // OpenGL ES 3.1
82 };
83 } else if (instance.gpu_mode() == kGpuModeDrmVirgl) {
84 bootconfig_args = {
85 {"androidboot.cpuvulkan.version", "0"},
86 {"androidboot.hardware.gralloc", "minigbm"},
87 {"androidboot.hardware.hwcomposer", "ranchu"},
88 {"androidboot.hardware.hwcomposer.mode", "client"},
89 {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
90 {"androidboot.hardware.hwcomposer.display_framebuffer_format",
91 instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
92 {"androidboot.hardware.egl", "mesa"},
93 // No "hardware" Vulkan support, yet
94 {"androidboot.opengles.version", "196608"}, // OpenGL ES 3.0
95 };
96 } else if (instance.gpu_mode() == kGpuModeGfxstream ||
97 instance.gpu_mode() == kGpuModeGfxstreamGuestAngle ||
98 instance.gpu_mode() ==
99 kGpuModeGfxstreamGuestAngleHostSwiftShader) {
100 const bool uses_angle =
101 instance.gpu_mode() == kGpuModeGfxstreamGuestAngle ||
102 instance.gpu_mode() == kGpuModeGfxstreamGuestAngleHostSwiftShader;
103
104 const std::string gles_impl = uses_angle ? "angle" : "emulation";
105
106 const std::string gfxstream_transport = instance.gpu_gfxstream_transport();
107 CF_EXPECT(gfxstream_transport == "virtio-gpu-asg" ||
108 gfxstream_transport == "virtio-gpu-pipe",
109 "Invalid Gfxstream transport option: \"" << gfxstream_transport
110 << "\"");
111
112 bootconfig_args = {
113 {"androidboot.cpuvulkan.version", "0"},
114 {"androidboot.hardware.gralloc", "minigbm"},
115 {"androidboot.hardware.hwcomposer", instance.hwcomposer()},
116 {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
117 {"androidboot.hardware.hwcomposer.display_framebuffer_format",
118 instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
119 {"androidboot.hardware.egl", gles_impl},
120 {"androidboot.hardware.vulkan", "ranchu"},
121 {"androidboot.hardware.gltransport", gfxstream_transport},
122 {"androidboot.opengles.version", "196609"}, // OpenGL ES 3.1
123 };
124 } else if (instance.gpu_mode() == kGpuModeCustom) {
125 bootconfig_args = {
126 {"androidboot.cpuvulkan.version", "0"},
127 {"androidboot.hardware.gralloc", "minigbm"},
128 {"androidboot.hardware.hwcomposer", instance.hwcomposer()},
129 {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
130 {"androidboot.hardware.hwcomposer.display_framebuffer_format",
131 instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
132 {"androidboot.hardware.egl", "angle"},
133 {"androidboot.hardware.vulkan", instance.guest_vulkan_driver()},
134 {"androidboot.hardware.gltransport", "virtio-gpu-asg"},
135 {"androidboot.opengles.version", "196609"}, // OpenGL ES 3.1
136 };
137 } else if (instance.gpu_mode() == kGpuModeNone) {
138 return {};
139 } else {
140 return CF_ERR("Unknown GPU mode " << instance.gpu_mode());
141 }
142
143 if (!instance.gpu_angle_feature_overrides_enabled().empty()) {
144 bootconfig_args["androidboot.hardware.angle_feature_overrides_enabled"] =
145 instance.gpu_angle_feature_overrides_enabled();
146 }
147 if (!instance.gpu_angle_feature_overrides_disabled().empty()) {
148 bootconfig_args["androidboot.hardware.angle_feature_overrides_disabled"] =
149 instance.gpu_angle_feature_overrides_disabled();
150 }
151
152 return bootconfig_args;
153 }
154
155 Result<std::unordered_map<std::string, std::string>>
ConfigureBootDevices(const CuttlefishConfig::InstanceSpecific & instance)156 CrosvmManager::ConfigureBootDevices(
157 const CuttlefishConfig::InstanceSpecific& instance) {
158 const int num_disks = instance.virtual_disk_paths().size();
159 const bool has_gpu = instance.hwcomposer() != kHwComposerNone;
160 // TODO There is no way to control this assignment with crosvm (yet)
161 if (HostArch() == Arch::X86_64) {
162 int num_gpu_pcis = has_gpu ? 1 : 0;
163 if (instance.gpu_mode() != kGpuModeNone &&
164 !instance.enable_gpu_vhost_user()) {
165 // crosvm has an additional PCI device for an ISA bridge when running
166 // with a gpu and without vhost user gpu.
167 num_gpu_pcis += 1;
168 }
169 // virtio_gpu and virtio_wl precedes the first console or disk
170 return ConfigureMultipleBootDevices("pci0000:00/0000:00:", 1 + num_gpu_pcis,
171 num_disks);
172 } else {
173 // On ARM64 crosvm, block devices are on their own bridge, so we don't
174 // need to calculate it, and the path is always the same
175 return {{{"androidboot.boot_devices", "10000.pci"}}};
176 }
177 }
178
ToSingleLineString(const Json::Value & value)179 std::string ToSingleLineString(const Json::Value& value) {
180 Json::StreamWriterBuilder builder;
181 builder["indentation"] = "";
182 return Json::writeString(builder, value);
183 }
184
MaybeConfigureVulkanIcd(const CuttlefishConfig & config,Command * command)185 void MaybeConfigureVulkanIcd(const CuttlefishConfig& config, Command* command) {
186 const auto& gpu_mode = config.ForDefaultInstance().gpu_mode();
187 if (gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader) {
188 // See https://github.com/KhronosGroup/Vulkan-Loader.
189 const std::string swiftshader_icd_json =
190 HostUsrSharePath("vulkan/icd.d/vk_swiftshader_icd.json");
191 command->AddEnvironmentVariable("VK_DRIVER_FILES", swiftshader_icd_json);
192 command->AddEnvironmentVariable("VK_ICD_FILENAMES", swiftshader_icd_json);
193 }
194 }
195
CrosvmPathForVhostUserGpu(const CuttlefishConfig & config)196 Result<std::string> CrosvmPathForVhostUserGpu(const CuttlefishConfig& config) {
197 const auto& instance = config.ForDefaultInstance();
198 switch (HostArch()) {
199 case Arch::Arm64:
200 return HostBinaryPath("aarch64-linux-gnu/crosvm");
201 case Arch::X86:
202 case Arch::X86_64:
203 return instance.crosvm_binary();
204 default:
205 break;
206 }
207 return CF_ERR("Unhandled host arch " << HostArchStr()
208 << " for vhost user gpu crosvm");
209 }
210
211 struct VhostUserDeviceCommands {
212 Command device_cmd;
213 Command device_logs_cmd;
214 };
BuildVhostUserGpu(const CuttlefishConfig & config,Command * main_crosvm_cmd)215 Result<VhostUserDeviceCommands> BuildVhostUserGpu(
216 const CuttlefishConfig& config, Command* main_crosvm_cmd) {
217 const auto& instance = config.ForDefaultInstance();
218 if (!instance.enable_gpu_vhost_user()) {
219 return CF_ERR("Attempting to build vhost user gpu when not enabled?");
220 }
221
222 auto gpu_device_socket_path =
223 instance.PerInstanceInternalUdsPath("vhost-user-gpu-socket");
224 auto gpu_device_logs_path =
225 instance.PerInstanceInternalPath("crosvm_vhost_user_gpu.fifo");
226 auto gpu_device_logs = CF_EXPECT(SharedFD::Fifo(gpu_device_logs_path, 0666));
227
228 Command gpu_device_logs_cmd(HostBinaryPath("log_tee"));
229 gpu_device_logs_cmd.AddParameter("--process_name=crosvm_gpu");
230 gpu_device_logs_cmd.AddParameter("--log_fd_in=", gpu_device_logs);
231 gpu_device_logs_cmd.SetStopper(KillSubprocessFallback([](Subprocess* proc) {
232 // Ask nicely so that log_tee gets a chance to process all the logs.
233 // TODO: b/335934714 - Make sure the process actually exits
234 bool res = kill(proc->pid(), SIGINT) == 0;
235 return res ? StopperResult::kStopSuccess : StopperResult::kStopFailure;
236 }));
237
238 const std::string crosvm_path = CF_EXPECT(CrosvmPathForVhostUserGpu(config));
239
240 CrosvmBuilder gpu_device_cmd;
241
242 // NOTE: The "main" crosvm process returns a kCrosvmVmResetExitCode when the
243 // guest exits but the "gpu" crosvm just exits cleanly with 0 after the "main"
244 // crosvm disconnects.
245 gpu_device_cmd.ApplyProcessRestarter(crosvm_path,
246 /*first_time_argument=*/"",
247 /*exit_code=*/0);
248
249 gpu_device_cmd.Cmd().AddParameter("device");
250 gpu_device_cmd.Cmd().AddParameter("gpu");
251
252 const auto& gpu_mode = instance.gpu_mode();
253 CF_EXPECT(
254 gpu_mode == kGpuModeGfxstream ||
255 gpu_mode == kGpuModeGfxstreamGuestAngle ||
256 gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader,
257 "GPU mode " << gpu_mode << " not yet supported with vhost user gpu.");
258
259 const std::string gpu_pci_address =
260 fmt::format("00:{:0>2x}.0", VmManager::kGpuPciSlotNum);
261
262 // Why does this need JSON instead of just following the normal flags style...
263 Json::Value gpu_params_json;
264 gpu_params_json["pci-address"] = gpu_pci_address;
265 if (gpu_mode == kGpuModeGfxstream) {
266 gpu_params_json["context-types"] = "gfxstream-gles:gfxstream-vulkan";
267 gpu_params_json["egl"] = true;
268 gpu_params_json["gles"] = true;
269 } else if (gpu_mode == kGpuModeGfxstreamGuestAngle ||
270 gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader) {
271 gpu_params_json["context-types"] = "gfxstream-vulkan";
272 gpu_params_json["egl"] = false;
273 gpu_params_json["gles"] = false;
274 }
275 gpu_params_json["glx"] = false;
276 gpu_params_json["surfaceless"] = true;
277 gpu_params_json["external-blob"] = instance.enable_gpu_external_blob();
278 gpu_params_json["system-blob"] = instance.enable_gpu_system_blob();
279
280 if (instance.hwcomposer() != kHwComposerNone) {
281 // "displays": [
282 // {
283 // "mode": {
284 // "windowed": [
285 // 720,
286 // 1280
287 // ]
288 // },
289 // "dpi": [
290 // 320,
291 // 320
292 // ],
293 // "refresh-rate": 60
294 // }
295 // ]
296 Json::Value displays(Json::arrayValue);
297 for (const auto& display_config : instance.display_configs()) {
298 Json::Value display_mode_windowed(Json::arrayValue);
299 display_mode_windowed[0] = display_config.width;
300 display_mode_windowed[1] = display_config.height;
301
302 Json::Value display_mode;
303 display_mode["windowed"] = display_mode_windowed;
304
305 Json::Value display_dpi(Json::arrayValue);
306 display_dpi[0] = display_config.dpi;
307 display_dpi[1] = display_config.dpi;
308
309 Json::Value display;
310 display["mode"] = display_mode;
311 display["dpi"] = display_dpi;
312 display["refresh-rate"] = display_config.refresh_rate_hz;
313
314 displays.append(display);
315 }
316 gpu_params_json["displays"] = displays;
317
318 gpu_device_cmd.Cmd().AddParameter("--wayland-sock=",
319 instance.frames_socket_path());
320 }
321
322 // Connect device to main crosvm:
323 gpu_device_cmd.Cmd().AddParameter("--socket=", gpu_device_socket_path);
324
325 main_crosvm_cmd->AddPrerequisite([gpu_device_socket_path]() -> Result<void> {
326 #ifdef __linux__
327 return WaitForUnixSocketListeningWithoutConnect(gpu_device_socket_path,
328 /*timeoutSec=*/30);
329 #else
330 return CF_ERR("Unhandled check if vhost user gpu ready.");
331 #endif
332 });
333 main_crosvm_cmd->AddParameter(
334 "--vhost-user=gpu,pci-address=", gpu_pci_address,
335 ",socket=", gpu_device_socket_path);
336
337 gpu_device_cmd.Cmd().AddParameter("--params");
338 gpu_device_cmd.Cmd().AddParameter(ToSingleLineString(gpu_params_json));
339
340 MaybeConfigureVulkanIcd(config, &gpu_device_cmd.Cmd());
341
342 gpu_device_cmd.Cmd().RedirectStdIO(Subprocess::StdIOChannel::kStdOut,
343 gpu_device_logs);
344 gpu_device_cmd.Cmd().RedirectStdIO(Subprocess::StdIOChannel::kStdErr,
345 gpu_device_logs);
346
347 return VhostUserDeviceCommands{
348 .device_cmd = std::move(gpu_device_cmd.Cmd()),
349 .device_logs_cmd = std::move(gpu_device_logs_cmd),
350 };
351 }
352
ConfigureGpu(const CuttlefishConfig & config,Command * crosvm_cmd)353 Result<void> ConfigureGpu(const CuttlefishConfig& config, Command* crosvm_cmd) {
354 const auto& instance = config.ForDefaultInstance();
355 const auto& gpu_mode = instance.gpu_mode();
356
357 const std::string gles_string =
358 gpu_mode == kGpuModeGfxstreamGuestAngle ||
359 gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader
360 ? ",gles=false"
361 : ",gles=true";
362
363 // 256MB so it is small enough for a 32-bit kernel.
364 const bool target_is_32bit = instance.target_arch() == Arch::Arm ||
365 instance.target_arch() == Arch::X86;
366 const std::string gpu_pci_bar_size =
367 target_is_32bit ? ",pci-bar-size=268435456" : "";
368
369 const std::string gpu_udmabuf_string =
370 instance.enable_gpu_udmabuf() ? ",udmabuf=true" : "";
371
372 const std::string gpu_renderer_features = instance.gpu_renderer_features();
373 const std::string gpu_renderer_features_param =
374 !gpu_renderer_features.empty()
375 ? ",renderer-features=\"" + gpu_renderer_features + "\""
376 : "";
377
378 const std::string gpu_common_string =
379 fmt::format(",pci-address=00:{:0>2x}.0", VmManager::kGpuPciSlotNum) +
380 gpu_udmabuf_string + gpu_pci_bar_size;
381 const std::string gpu_common_3d_string =
382 gpu_common_string + ",egl=true,surfaceless=true,glx=false" + gles_string +
383 gpu_renderer_features_param;
384
385 if (gpu_mode == kGpuModeGuestSwiftshader) {
386 crosvm_cmd->AddParameter("--gpu=backend=2D", gpu_common_string);
387 } else if (gpu_mode == kGpuModeDrmVirgl) {
388 crosvm_cmd->AddParameter("--gpu=backend=virglrenderer,context-types=virgl2",
389 gpu_common_3d_string);
390 } else if (gpu_mode == kGpuModeGfxstream) {
391 crosvm_cmd->AddParameter(
392 "--gpu=context-types=gfxstream-gles:gfxstream-vulkan:gfxstream-"
393 "composer",
394 gpu_common_3d_string);
395 } else if (gpu_mode == kGpuModeGfxstreamGuestAngle ||
396 gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader) {
397 crosvm_cmd->AddParameter(
398 "--gpu=context-types=gfxstream-vulkan:gfxstream-composer",
399 gpu_common_3d_string);
400 } else if (gpu_mode == kGpuModeCustom) {
401 const std::string gpu_context_types =
402 "--gpu=context-types=" + instance.gpu_context_types();
403 crosvm_cmd->AddParameter(gpu_context_types, gpu_common_string);
404 }
405
406 MaybeConfigureVulkanIcd(config, crosvm_cmd);
407
408 if (instance.hwcomposer() != kHwComposerNone) {
409 for (const auto& display_config : instance.display_configs()) {
410 const auto display_w = std::to_string(display_config.width);
411 const auto display_h = std::to_string(display_config.height);
412 const auto display_dpi = std::to_string(display_config.dpi);
413 const auto display_rr = std::to_string(display_config.refresh_rate_hz);
414 const auto display_params = android::base::Join(
415 std::vector<std::string>{
416 "mode=windowed[" + display_w + "," + display_h + "]",
417 "dpi=[" + display_dpi + "," + display_dpi + "]",
418 "refresh-rate=" + display_rr,
419 },
420 ",");
421
422 crosvm_cmd->AddParameter("--gpu-display=", display_params);
423 }
424
425 crosvm_cmd->AddParameter("--wayland-sock=", instance.frames_socket_path());
426 }
427
428 return {};
429 }
430
StartCommands(const CuttlefishConfig & config,std::vector<VmmDependencyCommand * > & dependencyCommands)431 Result<std::vector<MonitorCommand>> CrosvmManager::StartCommands(
432 const CuttlefishConfig& config,
433 std::vector<VmmDependencyCommand*>& dependencyCommands) {
434 auto instance = config.ForDefaultInstance();
435 auto environment = config.ForDefaultEnvironment();
436
437 CrosvmBuilder crosvm_cmd;
438 crosvm_cmd.Cmd().AddPrerequisite([&dependencyCommands]() -> Result<void> {
439 for (auto dependencyCommand : dependencyCommands) {
440 CF_EXPECT(dependencyCommand->WaitForAvailability());
441 }
442
443 return {};
444 });
445
446 // Add "--restore_path=<guest snapshot directory>" if there is a snapshot
447 // path supplied.
448 //
449 // Use the process_restarter "-first_time_argument" flag to only do this for
450 // the first invocation. If the guest requests a restart, we don't want crosvm
451 // to restore again. It should reboot normally.
452 std::string first_time_argument;
453 if (IsRestoring(config)) {
454 const std::string snapshot_dir_path = config.snapshot_path();
455 auto meta_info_json = CF_EXPECT(LoadMetaJson(snapshot_dir_path));
456 const std::vector<std::string> selectors{kGuestSnapshotField,
457 instance.id()};
458 const auto guest_snapshot_dir_suffix =
459 CF_EXPECT(GetValue<std::string>(meta_info_json, selectors));
460 // guest_snapshot_dir_suffix is a relative to
461 // the snapshot_path
462 const auto restore_path = snapshot_dir_path + "/" +
463 guest_snapshot_dir_suffix + "/" +
464 kGuestSnapshotBase;
465 first_time_argument = "--restore=" + restore_path;
466 }
467
468 crosvm_cmd.ApplyProcessRestarter(instance.crosvm_binary(),
469 first_time_argument, kCrosvmVmResetExitCode);
470 crosvm_cmd.Cmd().AddParameter("run");
471 crosvm_cmd.AddControlSocket(instance.CrosvmSocketPath(),
472 instance.crosvm_binary());
473
474 if (!instance.smt()) {
475 crosvm_cmd.Cmd().AddParameter("--no-smt");
476 }
477
478 // Disable USB passthrough. It isn't needed for any key use cases and it is
479 // not compatible with crosvm suspend-resume support yet (b/266622743).
480 // TODO: Allow it to be turned back on using a flag.
481 if (!instance.enable_usb()) {
482 crosvm_cmd.Cmd().AddParameter("--no-usb");
483 }
484
485 crosvm_cmd.Cmd().AddParameter("--core-scheduling=false");
486
487 if (instance.vhost_net()) {
488 crosvm_cmd.Cmd().AddParameter("--vhost-net");
489 }
490
491 if (config.virtio_mac80211_hwsim() &&
492 !environment.vhost_user_mac80211_hwsim().empty()) {
493 crosvm_cmd.Cmd().AddParameter("--vhost-user=mac80211-hwsim,socket=",
494 environment.vhost_user_mac80211_hwsim());
495 }
496
497 if (instance.protected_vm()) {
498 crosvm_cmd.Cmd().AddParameter("--protected-vm");
499 }
500
501 if (!instance.crosvm_use_balloon()) {
502 crosvm_cmd.Cmd().AddParameter("--no-balloon");
503 }
504
505 if (!instance.crosvm_use_rng()) {
506 crosvm_cmd.Cmd().AddParameter("--no-rng");
507 }
508
509 if (instance.gdb_port() > 0) {
510 CF_EXPECT(instance.cpus() == 1, "CPUs must be 1 for crosvm gdb mode");
511 crosvm_cmd.Cmd().AddParameter("--gdb=", instance.gdb_port());
512 }
513
514 std::optional<VhostUserDeviceCommands> vhost_user_gpu;
515 if (instance.enable_gpu_vhost_user()) {
516 vhost_user_gpu.emplace(
517 CF_EXPECT(BuildVhostUserGpu(config, &crosvm_cmd.Cmd())));
518 } else {
519 CF_EXPECT(ConfigureGpu(config, &crosvm_cmd.Cmd()));
520 }
521
522 if (instance.hwcomposer() != kHwComposerNone) {
523 const bool pmem_disabled = instance.mte() || !instance.use_pmem();
524 if (!pmem_disabled && FileExists(instance.hwcomposer_pmem_path())) {
525 crosvm_cmd.Cmd().AddParameter("--rw-pmem-device=",
526 instance.hwcomposer_pmem_path());
527 }
528 }
529
530 const auto gpu_capture_enabled = !instance.gpu_capture_binary().empty();
531
532 // crosvm_cmd.Cmd().AddParameter("--null-audio");
533 crosvm_cmd.Cmd().AddParameter("--mem=", instance.memory_mb());
534 crosvm_cmd.Cmd().AddParameter("--cpus=", instance.cpus());
535 if (instance.mte()) {
536 crosvm_cmd.Cmd().AddParameter("--mte");
537 }
538
539 auto disk_num = instance.virtual_disk_paths().size();
540 CF_EXPECT(VmManager::kMaxDisks >= disk_num,
541 "Provided too many disks (" << disk_num << "), maximum "
542 << VmManager::kMaxDisks << "supported");
543 for (const auto& disk : instance.virtual_disk_paths()) {
544 if (instance.protected_vm()) {
545 crosvm_cmd.AddReadOnlyDisk(disk);
546 } else {
547 crosvm_cmd.AddReadWriteDisk(disk);
548 }
549 }
550
551 if (instance.enable_webrtc()) {
552 bool is_chromeos =
553 instance.boot_flow() ==
554 CuttlefishConfig::InstanceSpecific::BootFlow::ChromeOs ||
555 instance.boot_flow() ==
556 CuttlefishConfig::InstanceSpecific::BootFlow::ChromeOsDisk;
557 auto touch_type_parameter =
558 is_chromeos ? "single-touch" : "multi-touch";
559
560 auto display_configs = instance.display_configs();
561 CF_EXPECT(display_configs.size() >= 1);
562
563 int touch_idx = 0;
564 for (auto& display_config : display_configs) {
565 crosvm_cmd.Cmd().AddParameter(
566 "--input=", touch_type_parameter, "[path=",
567 instance.touch_socket_path(touch_idx++),
568 ",width=", display_config.width,
569 ",height=", display_config.height, "]");
570 }
571 auto touchpad_configs = instance.touchpad_configs();
572 for (int i = 0; i < touchpad_configs.size(); ++i) {
573 auto touchpad_config = touchpad_configs[i];
574 crosvm_cmd.Cmd().AddParameter(
575 "--input=", touch_type_parameter, "[path=",
576 instance.touch_socket_path(touch_idx++),
577 ",width=", touchpad_config.width,
578 ",height=", touchpad_config.height,
579 ",name=", kTouchpadDefaultPrefix, i, "]");
580 }
581 crosvm_cmd.Cmd().AddParameter("--input=rotary[path=",
582 instance.rotary_socket_path(), "]");
583 crosvm_cmd.Cmd().AddParameter("--input=keyboard[path=",
584 instance.keyboard_socket_path(), "]");
585 crosvm_cmd.Cmd().AddParameter("--input=switches[path=",
586 instance.switches_socket_path(), "]");
587 }
588
589 SharedFD wifi_tap;
590 // GPU capture can only support named files and not file descriptors due to
591 // having to pass arguments to crosvm via a wrapper script.
592 #ifdef __linux__
593 if (!gpu_capture_enabled) {
594 // The PCI ordering of tap devices is important. Make sure any change here
595 // is reflected in ethprime u-boot variable.
596 // TODO(b/218364216, b/322862402): Crosvm occupies 32 PCI devices first and only then uses PCI
597 // functions which may break order. The final solution is going to be a PCI allocation strategy
598 // that will guarantee the ordering. For now, hardcode PCI network devices to unoccupied
599 // functions.
600 const pci::Address mobile_pci = pci::Address(0, VmManager::kNetPciDeviceNum, 1);
601 const pci::Address ethernet_pci = pci::Address(0, VmManager::kNetPciDeviceNum, 2);
602 crosvm_cmd.AddTap(instance.mobile_tap_name(), instance.mobile_mac(), mobile_pci);
603 crosvm_cmd.AddTap(instance.ethernet_tap_name(), instance.ethernet_mac(), ethernet_pci);
604
605 if (!config.virtio_mac80211_hwsim() && environment.enable_wifi()) {
606 wifi_tap = crosvm_cmd.AddTap(instance.wifi_tap_name());
607 }
608 }
609 #endif
610
611 const bool pmem_disabled = instance.mte() || !instance.use_pmem();
612 if (!pmem_disabled && FileExists(instance.access_kregistry_path())) {
613 crosvm_cmd.Cmd().AddParameter("--rw-pmem-device=",
614 instance.access_kregistry_path());
615 }
616
617 if (!pmem_disabled && FileExists(instance.pstore_path())) {
618 crosvm_cmd.Cmd().AddParameter("--pstore=path=", instance.pstore_path(),
619 ",size=", FileSize(instance.pstore_path()));
620 }
621
622 if (instance.enable_sandbox()) {
623 const bool seccomp_exists = DirectoryExists(instance.seccomp_policy_dir());
624 const std::string& var_empty_dir = kCrosvmVarEmptyDir;
625 const bool var_empty_available = DirectoryExists(var_empty_dir);
626 CF_EXPECT(var_empty_available && seccomp_exists,
627 var_empty_dir << " is not an existing, empty directory."
628 << "seccomp-policy-dir, "
629 << instance.seccomp_policy_dir()
630 << " does not exist");
631 crosvm_cmd.Cmd().AddParameter("--seccomp-policy-dir=",
632 instance.seccomp_policy_dir());
633 } else {
634 crosvm_cmd.Cmd().AddParameter("--disable-sandbox");
635 }
636
637 if (instance.vsock_guest_cid() >= 2) {
638 if (instance.vhost_user_vsock()) {
639 auto param =
640 fmt::format("/tmp/vsock_{}_{}/vhost.socket,max-queue-size=256",
641 instance.vsock_guest_cid(), std::to_string(getuid()));
642 crosvm_cmd.Cmd().AddParameter("--vhost-user=vsock,socket=", param);
643 } else {
644 crosvm_cmd.Cmd().AddParameter("--cid=", instance.vsock_guest_cid());
645 }
646 }
647
648 // /dev/hvc0 = kernel console
649 // If kernel log is enabled, the virtio-console port will be specified as
650 // a true console for Linux, and kernel messages will be printed there.
651 // Otherwise, the port will still be set up for bootloader and userspace
652 // messages, but the kernel will not print anything here. This keeps our
653 // kernel log event features working. If an alternative "earlycon" boot
654 // console is configured below on a legacy serial port, it will control
655 // the main log until the virtio-console takes over.
656 crosvm_cmd.AddHvcReadOnly(instance.kernel_log_pipe_name(),
657 instance.enable_kernel_log());
658
659 // /dev/hvc1 = serial console
660 if (instance.console()) {
661 // stdin is the only currently supported way to write data to a serial port
662 // in crosvm. A file (named pipe) is used here instead of stdout to ensure
663 // only the serial port output is received by the console forwarder as
664 // crosvm may print other messages to stdout.
665 if (instance.kgdb() || instance.use_bootloader()) {
666 crosvm_cmd.AddSerialConsoleReadWrite(instance.console_out_pipe_name(),
667 instance.console_in_pipe_name(),
668 instance.enable_kernel_log());
669 // In kgdb mode, we have the interactive console on ttyS0 (both Android's
670 // console and kdb), so we can disable the virtio-console port usually
671 // allocated to Android's serial console, and redirect it to a sink. This
672 // ensures that that the PCI device assignments (and thus sepolicy) don't
673 // have to change
674 crosvm_cmd.AddHvcSink();
675 } else {
676 crosvm_cmd.AddSerialSink();
677 crosvm_cmd.AddHvcReadWrite(instance.console_out_pipe_name(),
678 instance.console_in_pipe_name());
679 }
680 } else {
681 // Use an 8250 UART (ISA or platform device) for earlycon, as the
682 // virtio-console driver may not be available for early messages
683 // In kgdb mode, earlycon is an interactive console, and so early
684 // dmesg will go there instead of the kernel.log
685 if (instance.enable_kernel_log() &&
686 (instance.kgdb() || instance.use_bootloader())) {
687 crosvm_cmd.AddSerialConsoleReadOnly(instance.kernel_log_pipe_name());
688 }
689
690 // as above, create a fake virtio-console 'sink' port when the serial
691 // console is disabled, so the PCI device ID assignments don't move
692 // around
693 crosvm_cmd.AddHvcSink();
694 }
695
696 auto crosvm_logs_path = instance.PerInstanceInternalPath("crosvm.fifo");
697 auto crosvm_logs = CF_EXPECT(SharedFD::Fifo(crosvm_logs_path, 0666));
698
699 Command crosvm_log_tee_cmd(HostBinaryPath("log_tee"));
700 crosvm_log_tee_cmd.AddParameter("--process_name=crosvm");
701 crosvm_log_tee_cmd.AddParameter("--log_fd_in=", crosvm_logs);
702 crosvm_log_tee_cmd.SetStopper(KillSubprocessFallback([](Subprocess* proc) {
703 // Ask nicely so that log_tee gets a chance to process all the logs.
704 bool res = kill(proc->pid(), SIGINT) == 0;
705 // TODO: b/335934714 - Make sure the process actually exits
706 return res ? StopperResult::kStopSuccess : StopperResult::kStopFailure;
707 }));
708
709 // /dev/hvc2 = serial logging
710 // Serial port for logcat, redirected to a pipe
711 crosvm_cmd.AddHvcReadOnly(instance.logcat_pipe_name());
712
713 // /dev/hvc3 = keymaster (C++ implementation)
714 crosvm_cmd.AddHvcReadWrite(
715 instance.PerInstanceInternalPath("keymaster_fifo_vm.out"),
716 instance.PerInstanceInternalPath("keymaster_fifo_vm.in"));
717 // /dev/hvc4 = gatekeeper
718 crosvm_cmd.AddHvcReadWrite(
719 instance.PerInstanceInternalPath("gatekeeper_fifo_vm.out"),
720 instance.PerInstanceInternalPath("gatekeeper_fifo_vm.in"));
721
722 // /dev/hvc5 = bt
723 if (config.enable_host_bluetooth()) {
724 crosvm_cmd.AddHvcReadWrite(
725 instance.PerInstanceInternalPath("bt_fifo_vm.out"),
726 instance.PerInstanceInternalPath("bt_fifo_vm.in"));
727 } else {
728 crosvm_cmd.AddHvcSink();
729 }
730
731 // /dev/hvc6 = gnss
732 // /dev/hvc7 = location
733 if (instance.enable_gnss_grpc_proxy()) {
734 crosvm_cmd.AddHvcReadWrite(
735 instance.PerInstanceInternalPath("gnsshvc_fifo_vm.out"),
736 instance.PerInstanceInternalPath("gnsshvc_fifo_vm.in"));
737 crosvm_cmd.AddHvcReadWrite(
738 instance.PerInstanceInternalPath("locationhvc_fifo_vm.out"),
739 instance.PerInstanceInternalPath("locationhvc_fifo_vm.in"));
740 } else {
741 for (auto i = 0; i < 2; i++) {
742 crosvm_cmd.AddHvcSink();
743 }
744 }
745
746 // /dev/hvc8 = confirmationui
747 crosvm_cmd.AddHvcReadWrite(
748 instance.PerInstanceInternalPath("confui_fifo_vm.out"),
749 instance.PerInstanceInternalPath("confui_fifo_vm.in"));
750
751 // /dev/hvc9 = uwb
752 if (config.enable_host_uwb()) {
753 crosvm_cmd.AddHvcReadWrite(
754 instance.PerInstanceInternalPath("uwb_fifo_vm.out"),
755 instance.PerInstanceInternalPath("uwb_fifo_vm.in"));
756 } else {
757 crosvm_cmd.AddHvcSink();
758 }
759
760 // /dev/hvc10 = oemlock
761 crosvm_cmd.AddHvcReadWrite(
762 instance.PerInstanceInternalPath("oemlock_fifo_vm.out"),
763 instance.PerInstanceInternalPath("oemlock_fifo_vm.in"));
764
765 // /dev/hvc11 = keymint (Rust implementation)
766 crosvm_cmd.AddHvcReadWrite(
767 instance.PerInstanceInternalPath("keymint_fifo_vm.out"),
768 instance.PerInstanceInternalPath("keymint_fifo_vm.in"));
769
770 // /dev/hvc12 = NFC
771 if (config.enable_host_nfc()) {
772 crosvm_cmd.AddHvcReadWrite(
773 instance.PerInstanceInternalPath("nfc_fifo_vm.out"),
774 instance.PerInstanceInternalPath("nfc_fifo_vm.in"));
775 } else {
776 crosvm_cmd.AddHvcSink();
777 }
778
779
780 // /dev/hvc13 = sensors
781 crosvm_cmd.AddHvcReadWrite(
782 instance.PerInstanceInternalPath("sensors_fifo_vm.out"),
783 instance.PerInstanceInternalPath("sensors_fifo_vm.in"));
784
785 // /dev/hvc14 = MCU CONTROL
786 if (instance.mcu()["control"]["type"].asString() == "serial") {
787 auto path = instance.PerInstanceInternalPath("mcu");
788 path += "/" + instance.mcu()["control"]["path"].asString();
789 crosvm_cmd.AddHvcReadWrite(path, path);
790 } else {
791 crosvm_cmd.AddHvcSink();
792 }
793
794 // /dev/hvc15 = MCU UART
795 if (instance.mcu()["uart0"]["type"].asString() == "serial") {
796 auto path = instance.PerInstanceInternalPath("mcu");
797 path += "/" + instance.mcu()["uart0"]["path"].asString();
798 crosvm_cmd.AddHvcReadWrite(path, path);
799 } else {
800 crosvm_cmd.AddHvcSink();
801 }
802
803 for (auto i = 0; i < VmManager::kMaxDisks - disk_num; i++) {
804 crosvm_cmd.AddHvcSink();
805 }
806 CF_EXPECT(crosvm_cmd.HvcNum() + disk_num ==
807 VmManager::kMaxDisks + VmManager::kDefaultNumHvcs,
808 "HVC count (" << crosvm_cmd.HvcNum() << ") + disk count ("
809 << disk_num << ") is not the expected total of "
810 << VmManager::kMaxDisks + VmManager::kDefaultNumHvcs
811 << " devices");
812
813 if (instance.enable_audio()) {
814 crosvm_cmd.Cmd().AddParameter("--sound=", instance.audio_server_path());
815 }
816
817 // TODO(b/162071003): virtiofs crashes without sandboxing, this should be
818 // fixed
819 if (instance.enable_virtiofs()) {
820 CF_EXPECT(instance.enable_sandbox(),
821 "virtiofs is currently not supported without sandboxing");
822 // Set up directory shared with virtiofs
823 crosvm_cmd.Cmd().AddParameter(
824 "--shared-dir=", instance.PerInstancePath(kSharedDirName),
825 ":shared:type=fs");
826 }
827
828 if (instance.target_arch() == Arch::X86_64) {
829 crosvm_cmd.Cmd().AddParameter("--pflash=", instance.pflash_path());
830 }
831
832 // This needs to be the last parameter
833 crosvm_cmd.Cmd().AddParameter("--bios=", instance.bootloader());
834
835 std::vector<MonitorCommand> commands;
836
837 if (vhost_user_gpu) {
838 // The vhost user gpu crosvm command should be added before the main
839 // crosvm command so that the main crosvm command can use a prerequisite
840 // to wait for the communication socket to be ready.
841 commands.emplace_back(std::move(vhost_user_gpu->device_cmd));
842 commands.emplace_back(std::move(vhost_user_gpu->device_logs_cmd));
843 }
844
845 // log_tee must be added before crosvm_cmd to ensure all of crosvm's logs are
846 // captured during shutdown. Processes are stopped in reverse order.
847 commands.emplace_back(std::move(crosvm_log_tee_cmd));
848
849 if (gpu_capture_enabled) {
850 const std::string gpu_capture_basename =
851 cpp_basename(instance.gpu_capture_binary());
852
853 auto gpu_capture_logs_path =
854 instance.PerInstanceInternalPath("gpu_capture.fifo");
855 auto gpu_capture_logs =
856 CF_EXPECT(SharedFD::Fifo(gpu_capture_logs_path, 0666));
857
858 Command gpu_capture_log_tee_cmd(HostBinaryPath("log_tee"));
859 gpu_capture_log_tee_cmd.AddParameter("--process_name=",
860 gpu_capture_basename);
861 gpu_capture_log_tee_cmd.AddParameter("--log_fd_in=", gpu_capture_logs);
862
863 Command gpu_capture_command(instance.gpu_capture_binary());
864 if (gpu_capture_basename == "ngfx") {
865 // Crosvm depends on command line arguments being passed as multiple
866 // arguments but ngfx only allows a single `--args`. To work around this,
867 // create a wrapper script that launches crosvm with all of the arguments
868 // and pass this wrapper script to ngfx.
869 const std::string crosvm_wrapper_path =
870 instance.PerInstanceInternalPath("crosvm_wrapper.sh");
871 const std::string crosvm_wrapper_content =
872 crosvm_cmd.Cmd().AsBashScript(crosvm_logs_path);
873
874 CF_EXPECT(android::base::WriteStringToFile(crosvm_wrapper_content,
875 crosvm_wrapper_path));
876 CF_EXPECT(MakeFileExecutable(crosvm_wrapper_path));
877
878 gpu_capture_command.AddParameter("--exe=", crosvm_wrapper_path);
879 gpu_capture_command.AddParameter("--launch-detached");
880 gpu_capture_command.AddParameter("--verbose");
881 gpu_capture_command.AddParameter("--activity=Frame Debugger");
882 } else {
883 // TODO(natsu): renderdoc
884 return CF_ERR(
885 "Unhandled GPU capture binary: " << instance.gpu_capture_binary());
886 }
887
888 gpu_capture_command.RedirectStdIO(Subprocess::StdIOChannel::kStdOut,
889 gpu_capture_logs);
890 gpu_capture_command.RedirectStdIO(Subprocess::StdIOChannel::kStdErr,
891 gpu_capture_logs);
892
893 commands.emplace_back(std::move(gpu_capture_log_tee_cmd));
894 commands.emplace_back(std::move(gpu_capture_command));
895 } else {
896 crosvm_cmd.Cmd().RedirectStdIO(Subprocess::StdIOChannel::kStdOut,
897 crosvm_logs);
898 crosvm_cmd.Cmd().RedirectStdIO(Subprocess::StdIOChannel::kStdErr,
899 crosvm_logs);
900 commands.emplace_back(std::move(crosvm_cmd.Cmd()), true);
901 }
902
903 return commands;
904 }
905
WaitForRestoreComplete(SharedFD stop_fd) const906 Result<bool> CrosvmManager::WaitForRestoreComplete(SharedFD stop_fd) const {
907 auto instance = CF_EXPECT(CuttlefishConfig::Get())->ForDefaultInstance();
908
909 // Wait for the control socket to exist. It is created early in crosvm's
910 // startup sequence, but the process may not even have been exec'd by CF at
911 // this point.
912 while (!FileExists(instance.CrosvmSocketPath())) {
913 std::vector<PollSharedFd> poll = {{.fd = stop_fd, .events = POLLIN}};
914 const int result = SharedFD::Poll(poll, 50 /* ms */);
915 // Check for errors.
916 CF_EXPECT(result >= 0, "failed to wait on stop_fd: " << strerror(errno));
917 // Check if pipe became readable or closed.
918 if (result > 0) {
919 return false;
920 }
921 }
922
923 // Ask crosvm to resume the VM. crosvm promises to not complete this command
924 // until the vCPUs are started (even if it was never suspended to begin
925 // with).
926 auto infop = CF_EXPECT(Execute(
927 std::vector<std::string>{
928 instance.crosvm_binary(),
929 "resume",
930 instance.CrosvmSocketPath(),
931 "--full",
932 },
933 SubprocessOptions(), WEXITED));
934 CF_EXPECT_EQ(infop.si_code, CLD_EXITED);
935 CF_EXPECTF(infop.si_status == 0, "crosvm resume returns non zero code {}",
936 infop.si_status);
937 return true;
938 }
939
940 } // namespace vm_manager
941 } // namespace cuttlefish
942