1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "host/libs/vm_manager/qemu_manager.h"
18 
19 #include <string.h>
20 #include <sys/socket.h>
21 #include <sys/stat.h>
22 #include <sys/types.h>
23 #include <sys/un.h>
24 #include <sys/wait.h>
25 #include <unistd.h>
26 
27 #include <cstdlib>
28 #include <iomanip>
29 #include <string>
30 #include <unordered_map>
31 #include <utility>
32 #include <vector>
33 
34 #include <android-base/strings.h>
35 #include <android-base/logging.h>
36 #include <vulkan/vulkan.h>
37 
38 #include "common/libs/utils/files.h"
39 #include "common/libs/utils/result.h"
40 #include "common/libs/utils/subprocess.h"
41 #include "host/libs/config/command_source.h"
42 #include "host/libs/config/cuttlefish_config.h"
43 
44 namespace cuttlefish {
45 namespace vm_manager {
46 namespace {
47 
GetMonitorPath(const CuttlefishConfig & config)48 std::string GetMonitorPath(const CuttlefishConfig& config) {
49   return config.ForDefaultInstance().PerInstanceInternalUdsPath(
50       "qemu_monitor.sock");
51 }
52 
Stop()53 StopperResult Stop() {
54   auto config = CuttlefishConfig::Get();
55   auto monitor_path = GetMonitorPath(*config);
56   auto monitor_sock = SharedFD::SocketLocalClient(
57       monitor_path.c_str(), false, SOCK_STREAM);
58 
59   if (!monitor_sock->IsOpen()) {
60     LOG(ERROR) << "The connection to qemu is closed, is it still running?";
61     return StopperResult::kStopFailure;
62   }
63   char msg[] = "{\"execute\":\"qmp_capabilities\"}{\"execute\":\"quit\"}";
64   ssize_t len = sizeof(msg) - 1;
65   while (len > 0) {
66     int tmp = monitor_sock->Write(msg, len);
67     if (tmp < 0) {
68       LOG(ERROR) << "Error writing to socket: " << monitor_sock->StrError();
69       return StopperResult::kStopFailure;
70     }
71     len -= tmp;
72   }
73   // Log the reply
74   char buff[1000];
75   while ((len = monitor_sock->Read(buff, sizeof(buff) - 1)) > 0) {
76     buff[len] = '\0';
77     LOG(INFO) << "From qemu monitor: " << buff;
78   }
79 
80   return StopperResult::kStopSuccess;
81 }
82 
GetQemuVersion(const std::string & qemu_binary)83 Result<std::pair<int, int>> GetQemuVersion(const std::string& qemu_binary) {
84   Command qemu_version_cmd(qemu_binary);
85   qemu_version_cmd.AddParameter("-version");
86 
87   std::string qemu_version_input, qemu_version_output, qemu_version_error;
88   cuttlefish::SubprocessOptions options;
89   options.Verbose(false);
90   int qemu_version_ret = cuttlefish::RunWithManagedStdio(
91       std::move(qemu_version_cmd), &qemu_version_input, &qemu_version_output,
92       &qemu_version_error, std::move(options));
93   CF_EXPECT(qemu_version_ret == 0,
94             qemu_binary << " -version returned unexpected response "
95                         << qemu_version_output << ". Stderr was "
96                         << qemu_version_error);
97 
98   // Snip around the extra text we don't care about
99   qemu_version_output.erase(0, std::string("QEMU emulator version ").length());
100   auto space_pos = qemu_version_output.find(" ", 0);
101   if (space_pos != std::string::npos) {
102     qemu_version_output.resize(space_pos);
103   }
104 
105   auto qemu_version_bits = android::base::Split(qemu_version_output, ".");
106   return {{std::stoi(qemu_version_bits[0]), std::stoi(qemu_version_bits[1])}};
107 }
108 
109 }  // namespace
110 
QemuManager(Arch arch)111 QemuManager::QemuManager(Arch arch) : arch_(arch) {}
112 
IsSupported()113 bool QemuManager::IsSupported() {
114   return HostSupportsQemuCli();
115 }
116 
117 Result<std::unordered_map<std::string, std::string>>
ConfigureGraphics(const CuttlefishConfig::InstanceSpecific & instance)118 QemuManager::ConfigureGraphics(
119     const CuttlefishConfig::InstanceSpecific& instance) {
120   // Override the default HAL search paths in all cases. We do this because
121   // the HAL search path allows for fallbacks, and fallbacks in conjunction
122   // with properties lead to non-deterministic behavior while loading the
123   // HALs.
124 
125   std::unordered_map<std::string, std::string> bootconfig_args;
126   auto gpu_mode = instance.gpu_mode();
127   if (gpu_mode == kGpuModeGuestSwiftshader) {
128     bootconfig_args = {
129         {"androidboot.cpuvulkan.version", std::to_string(VK_API_VERSION_1_2)},
130         {"androidboot.hardware.gralloc", "minigbm"},
131         {"androidboot.hardware.hwcomposer", instance.hwcomposer()},
132         {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
133         {"androidboot.hardware.hwcomposer.display_framebuffer_format",
134          instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
135         {"androidboot.hardware.egl", "angle"},
136         {"androidboot.hardware.vulkan", "pastel"},
137         // OpenGL ES 3.1
138         {"androidboot.opengles.version", "196609"},
139     };
140   } else if (gpu_mode == kGpuModeDrmVirgl) {
141     bootconfig_args = {
142         {"androidboot.cpuvulkan.version", "0"},
143         {"androidboot.hardware.gralloc", "minigbm"},
144         {"androidboot.hardware.hwcomposer", "ranchu"},
145         {"androidboot.hardware.hwcomposer.mode", "client"},
146         {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
147         {"androidboot.hardware.hwcomposer.display_framebuffer_format",
148          instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
149         {"androidboot.hardware.egl", "mesa"},
150         // No "hardware" Vulkan support, yet
151         // OpenGL ES 3.0
152         {"androidboot.opengles.version", "196608"},
153     };
154   } else if (gpu_mode == kGpuModeGfxstream ||
155              gpu_mode == kGpuModeGfxstreamGuestAngle ||
156              gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader) {
157     const bool uses_angle =
158         gpu_mode == kGpuModeGfxstreamGuestAngle ||
159         gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader;
160     const std::string gles_impl = uses_angle ? "angle" : "emulation";
161     const std::string gltransport =
162         (instance.guest_android_version() == "11.0.0") ? "virtio-gpu-pipe"
163                                                        : "virtio-gpu-asg";
164     bootconfig_args = {
165         {"androidboot.cpuvulkan.version", "0"},
166         {"androidboot.hardware.gralloc", "minigbm"},
167         {"androidboot.hardware.hwcomposer", instance.hwcomposer()},
168         {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
169         {"androidboot.hardware.hwcomposer.display_framebuffer_format",
170          instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
171         {"androidboot.hardware.egl", gles_impl},
172         {"androidboot.hardware.vulkan", "ranchu"},
173         {"androidboot.hardware.gltransport", gltransport},
174         {"androidboot.opengles.version", "196609"},  // OpenGL ES 3.1
175     };
176   } else if (instance.gpu_mode() == kGpuModeNone) {
177     return {};
178   } else {
179     return CF_ERR("Unhandled GPU mode: " << instance.gpu_mode());
180   }
181 
182   if (!instance.gpu_angle_feature_overrides_enabled().empty()) {
183     bootconfig_args["androidboot.hardware.angle_feature_overrides_enabled"] =
184         instance.gpu_angle_feature_overrides_enabled();
185   }
186   if (!instance.gpu_angle_feature_overrides_disabled().empty()) {
187     bootconfig_args["androidboot.hardware.angle_feature_overrides_disabled"] =
188         instance.gpu_angle_feature_overrides_disabled();
189   }
190 
191   return bootconfig_args;
192 }
193 
194 Result<std::unordered_map<std::string, std::string>>
ConfigureBootDevices(const CuttlefishConfig::InstanceSpecific & instance)195 QemuManager::ConfigureBootDevices(
196     const CuttlefishConfig::InstanceSpecific& instance) {
197   const int num_disks = instance.virtual_disk_paths().size();
198   const int num_gpu = instance.hwcomposer() != kHwComposerNone;
199   switch (arch_) {
200     case Arch::Arm:
201       return {{{"androidboot.boot_devices", "3f000000.pcie"}}};
202     case Arch::Arm64:
203 #ifdef __APPLE__
204       return {{{"androidboot.boot_devices", "3f000000.pcie"}}};
205 #else
206       return {{{"androidboot.boot_devices", "4010000000.pcie"}}};
207 #endif
208     case Arch::RiscV64:
209       return {{{"androidboot.boot_devices", "soc/30000000.pci"}}};
210     case Arch::X86:
211     case Arch::X86_64: {
212       // QEMU has additional PCI devices for an ISA bridge and PIIX4
213       // virtio_gpu precedes the first console or disk
214       // TODO(schuffelen): Simplify this logic when crosvm uses multiport
215       int pci_offset = 3 + num_gpu - VmManager::kDefaultNumHvcs;
216       return ConfigureMultipleBootDevices("pci0000:00/0000:00:", pci_offset,
217                                           num_disks);
218     }
219   }
220 }
221 
StartCommands(const CuttlefishConfig & config,std::vector<VmmDependencyCommand * > &)222 Result<std::vector<MonitorCommand>> QemuManager::StartCommands(
223     const CuttlefishConfig& config, std::vector<VmmDependencyCommand*>&) {
224   auto instance = config.ForDefaultInstance();
225   std::string qemu_binary = instance.qemu_binary_dir();
226   switch (arch_) {
227     case Arch::Arm:
228       qemu_binary += "/qemu-system-arm";
229       break;
230     case Arch::Arm64:
231       qemu_binary += "/qemu-system-aarch64";
232       break;
233     case Arch::RiscV64:
234       qemu_binary += "/qemu-system-riscv64";
235       break;
236     case Arch::X86:
237       qemu_binary += "/qemu-system-i386";
238       break;
239     case Arch::X86_64:
240       qemu_binary += "/qemu-system-x86_64";
241       break;
242   }
243 
244   auto qemu_version = CF_EXPECT(GetQemuVersion(qemu_binary));
245   Command qemu_cmd(qemu_binary, KillSubprocessFallback(Stop));
246 
247   int hvc_num = 0;
248   int serial_num = 0;
249   auto add_hvc_sink = [&qemu_cmd, &hvc_num]() {
250     qemu_cmd.AddParameter("-chardev");
251     qemu_cmd.AddParameter("null,id=hvc", hvc_num);
252     qemu_cmd.AddParameter("-device");
253     qemu_cmd.AddParameter("virtconsole,bus=virtio-serial.0,chardev=hvc",
254                           hvc_num);
255     hvc_num++;
256   };
257   auto add_serial_sink = [&qemu_cmd, &serial_num]() {
258     qemu_cmd.AddParameter("-chardev");
259     qemu_cmd.AddParameter("null,id=serial", serial_num);
260     qemu_cmd.AddParameter("-serial");
261     qemu_cmd.AddParameter("chardev:serial", serial_num);
262     serial_num++;
263   };
264   auto add_serial_console_ro = [&qemu_cmd,
265                                 &serial_num](const std::string& output) {
266     qemu_cmd.AddParameter("-chardev");
267     qemu_cmd.AddParameter("file,id=serial", serial_num, ",path=", output,
268                           ",append=on");
269     qemu_cmd.AddParameter("-serial");
270     qemu_cmd.AddParameter("chardev:serial", serial_num);
271     serial_num++;
272   };
273   auto add_serial_console = [&qemu_cmd,
274                              &serial_num](const std::string& prefix) {
275     qemu_cmd.AddParameter("-chardev");
276     qemu_cmd.AddParameter("pipe,id=serial", serial_num, ",path=", prefix);
277     qemu_cmd.AddParameter("-serial");
278     qemu_cmd.AddParameter("chardev:serial", serial_num);
279     serial_num++;
280   };
281   auto add_hvc_ro = [&qemu_cmd, &hvc_num](const std::string& output) {
282     qemu_cmd.AddParameter("-chardev");
283     qemu_cmd.AddParameter("file,id=hvc", hvc_num, ",path=", output,
284                           ",append=on");
285     qemu_cmd.AddParameter("-device");
286     qemu_cmd.AddParameter("virtconsole,bus=virtio-serial.0,chardev=hvc",
287                           hvc_num);
288     hvc_num++;
289   };
290   auto add_hvc = [&qemu_cmd, &hvc_num](const std::string& prefix) {
291     qemu_cmd.AddParameter("-chardev");
292     qemu_cmd.AddParameter("pipe,id=hvc", hvc_num, ",path=", prefix);
293     qemu_cmd.AddParameter("-device");
294     qemu_cmd.AddParameter("virtconsole,bus=virtio-serial.0,chardev=hvc",
295                           hvc_num);
296     hvc_num++;
297   };
298   auto add_hvc_serial = [&qemu_cmd, &hvc_num](const std::string& prefix) {
299     qemu_cmd.AddParameter("-chardev");
300     qemu_cmd.AddParameter("serial,id=hvc", hvc_num, ",path=", prefix);
301     qemu_cmd.AddParameter("-device");
302     qemu_cmd.AddParameter("virtconsole,bus=virtio-serial.0,chardev=hvc",
303                           hvc_num);
304     hvc_num++;
305   };
306 
307   bool is_arm = arch_ == Arch::Arm || arch_ == Arch::Arm64;
308   bool is_x86 = arch_ == Arch::X86 || arch_ == Arch::X86_64;
309   bool is_riscv64 = arch_ == Arch::RiscV64;
310 
311   auto access_kregistry_size_bytes = 0;
312   if (FileExists(instance.access_kregistry_path())) {
313     access_kregistry_size_bytes = FileSize(instance.access_kregistry_path());
314     CF_EXPECT((access_kregistry_size_bytes & (1024 * 1024 - 1)) == 0,
315               instance.access_kregistry_path()
316                   << " file size (" << access_kregistry_size_bytes
317                   << ") not a multiple of 1MB");
318   }
319 
320   auto hwcomposer_pmem_size_bytes = 0;
321   if (instance.hwcomposer() != kHwComposerNone) {
322     if (FileExists(instance.hwcomposer_pmem_path())) {
323       hwcomposer_pmem_size_bytes = FileSize(instance.hwcomposer_pmem_path());
324       CF_EXPECT((hwcomposer_pmem_size_bytes & (1024 * 1024 - 1)) == 0,
325                 instance.hwcomposer_pmem_path()
326                     << " file size (" << hwcomposer_pmem_size_bytes
327                     << ") not a multiple of 1MB");
328     }
329   }
330 
331   auto pstore_size_bytes = 0;
332   if (FileExists(instance.pstore_path())) {
333     pstore_size_bytes = FileSize(instance.pstore_path());
334     CF_EXPECT((pstore_size_bytes & (1024 * 1024 - 1)) == 0,
335               instance.pstore_path() << " file size (" << pstore_size_bytes
336                                      << ") not a multiple of 1MB");
337   }
338 
339   qemu_cmd.AddParameter("-name");
340   qemu_cmd.AddParameter("guest=", instance.instance_name(), ",debug-threads=on");
341 
342   qemu_cmd.AddParameter("-machine");
343   std::string machine = is_x86 ? "pc,nvdimm=on" : "virt";
344   if (IsHostCompatible(arch_)) {
345 #ifdef __linux__
346     machine += ",accel=kvm";
347 #elif defined(__APPLE__)
348     machine += ",accel=hvf";
349 #else
350 #error "Unknown OS"
351 #endif
352     if (is_arm) {
353       machine += ",gic-version=3";
354     }
355   } else if (is_arm) {
356     // QEMU doesn't support GICv3 with TCG yet
357     machine += ",gic-version=2";
358     CF_EXPECT(instance.cpus() <= 8, "CPUs must be no more than 8 with GICv2");
359   }
360   if (instance.mte()) {
361     machine += ",mte=on";
362   }
363   qemu_cmd.AddParameter(machine, ",usb=off,dump-guest-core=off");
364 
365   qemu_cmd.AddParameter("-m");
366   auto maxmem = instance.memory_mb() +
367                 (access_kregistry_size_bytes / 1024 / 1024) +
368                 (hwcomposer_pmem_size_bytes / 1024 / 1024) +
369                 (is_x86 ? pstore_size_bytes / 1024 / 1024 : 0);
370   auto slots = is_x86 ? ",slots=2" : "";
371   qemu_cmd.AddParameter("size=", instance.memory_mb(), "M",
372                         ",maxmem=", maxmem, "M", slots);
373 
374   qemu_cmd.AddParameter("-overcommit");
375   qemu_cmd.AddParameter("mem-lock=off");
376 
377   // Assume SMT is always 2 threads per core, which is how most hardware
378   // today is configured, and the way crosvm does it
379   qemu_cmd.AddParameter("-smp");
380   if (instance.smt()) {
381     CF_EXPECT(instance.cpus() % 2 == 0,
382               "CPUs must be a multiple of 2 in SMT mode");
383     qemu_cmd.AddParameter(instance.cpus(), ",cores=",
384                           instance.cpus() / 2, ",threads=2");
385   } else {
386     qemu_cmd.AddParameter(instance.cpus(), ",cores=",
387                           instance.cpus(), ",threads=1");
388   }
389 
390   qemu_cmd.AddParameter("-uuid");
391   qemu_cmd.AddParameter(instance.uuid());
392 
393   qemu_cmd.AddParameter("-no-user-config");
394   qemu_cmd.AddParameter("-nodefaults");
395   qemu_cmd.AddParameter("-no-shutdown");
396 
397   qemu_cmd.AddParameter("-rtc");
398   qemu_cmd.AddParameter("base=utc");
399 
400   qemu_cmd.AddParameter("-boot");
401   qemu_cmd.AddParameter("strict=on");
402 
403   qemu_cmd.AddParameter("-chardev");
404   qemu_cmd.AddParameter("socket,id=charmonitor,path=", GetMonitorPath(config),
405                         ",server=on,wait=off");
406 
407   qemu_cmd.AddParameter("-mon");
408   qemu_cmd.AddParameter("chardev=charmonitor,id=monitor,mode=control");
409 
410   auto gpu_mode = instance.gpu_mode();
411   if (gpu_mode == kGpuModeDrmVirgl) {
412     qemu_cmd.AddParameter("-display");
413     qemu_cmd.AddParameter("egl-headless");
414 
415     qemu_cmd.AddParameter("-vnc");
416     qemu_cmd.AddParameter("127.0.0.1:", instance.qemu_vnc_server_port());
417   } else if (gpu_mode == kGpuModeGuestSwiftshader ||
418              gpu_mode == kGpuModeGfxstream ||
419              gpu_mode == kGpuModeGfxstreamGuestAngle ||
420              gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader) {
421     qemu_cmd.AddParameter("-vnc");
422     qemu_cmd.AddParameter("127.0.0.1:", instance.qemu_vnc_server_port());
423   } else {
424     qemu_cmd.AddParameter("-display");
425     qemu_cmd.AddParameter("none");
426   }
427 
428   if (instance.hwcomposer() != kHwComposerNone) {
429     auto display_configs = instance.display_configs();
430     CF_EXPECT(display_configs.size() >= 1);
431     auto display_config = display_configs[0];
432 
433     qemu_cmd.AddParameter("-device");
434 
435     std::string gpu_device;
436     if (gpu_mode == kGpuModeGuestSwiftshader || qemu_version.first < 6) {
437       gpu_device = "virtio-gpu-pci";
438     } else if (gpu_mode == kGpuModeDrmVirgl) {
439       gpu_device = "virtio-gpu-gl-pci";
440     } else if (gpu_mode == kGpuModeGfxstream) {
441       gpu_device =
442           "virtio-gpu-rutabaga,x-gfxstream-gles=on,gfxstream-vulkan=on,"
443           "x-gfxstream-composer=on,hostmem=256M";
444     } else if (gpu_mode == kGpuModeGfxstreamGuestAngle ||
445                gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader) {
446       gpu_device =
447           "virtio-gpu-rutabaga,gfxstream-vulkan=on,"
448           "x-gfxstream-composer=on,hostmem=256M";
449 
450       if (gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader) {
451         // See https://github.com/KhronosGroup/Vulkan-Loader.
452         const std::string swiftshader_icd_json =
453             HostUsrSharePath("vulkan/icd.d/vk_swiftshader_icd.json");
454         qemu_cmd.AddEnvironmentVariable("VK_DRIVER_FILES",
455                                         swiftshader_icd_json);
456         qemu_cmd.AddEnvironmentVariable("VK_ICD_FILENAMES",
457                                         swiftshader_icd_json);
458       }
459     }
460 
461     qemu_cmd.AddParameter(
462         gpu_device, ",id=gpu0",
463         fmt::format(",addr={:0>2x}.0", VmManager::kGpuPciSlotNum),
464         ",xres=", display_config.width, ",yres=", display_config.height);
465   }
466 
467   if (!instance.console()) {
468     // In kgdb mode, earlycon is an interactive console, and so early
469     // dmesg will go there instead of the kernel.log. On QEMU, we do this
470     // bit of logic up before the hvc console is set up, so the command line
471     // flags appear in the right order and "append=on" does the right thing
472     if (instance.enable_kernel_log() &&
473         (instance.kgdb() || instance.use_bootloader())) {
474       add_serial_console_ro(instance.kernel_log_pipe_name());
475     }
476   }
477 
478   qemu_cmd.AddParameter("-device");
479   qemu_cmd.AddParameter(
480       "virtio-serial-pci-non-transitional,max_ports=31,id=virtio-serial");
481 
482   // /dev/hvc0 = kernel console
483   // If kernel log is enabled, the virtio-console port will be specified as
484   // a true console for Linux, and kernel messages will be printed there.
485   // Otherwise, the port will still be set up for bootloader and userspace
486   // messages, but the kernel will not print anything here. This keeps our
487   // kernel log event features working. If an alternative "earlycon" boot
488   // console is configured above on a legacy serial port, it will control
489   // the main log until the virtio-console takes over.
490   // (Note that QEMU does not automatically generate console= parameters for
491   //  the bootloader/kernel cmdline, so the control of whether this pipe is
492   //  actually managed by the kernel as a console is handled elsewhere.)
493   add_hvc_ro(instance.kernel_log_pipe_name());
494 
495   // /dev/hvc1 = serial console
496   if (instance.console()) {
497     if (instance.kgdb() || instance.use_bootloader()) {
498       add_serial_console(instance.console_pipe_prefix());
499 
500       // In kgdb mode, we have the interactive console on ttyS0 (both Android's
501       // console and kdb), so we can disable the virtio-console port usually
502       // allocated to Android's serial console, and redirect it to a sink. This
503       // ensures that that the PCI device assignments (and thus sepolicy) don't
504       // have to change
505       add_hvc_sink();
506     } else {
507       add_serial_sink();
508       add_hvc(instance.console_pipe_prefix());
509     }
510   } else {
511     if (instance.kgdb() || instance.use_bootloader()) {
512       // The add_serial_console_ro() call above was applied by the time we reach
513       // this code, so we don't need another add_serial_*() call
514     }
515 
516     // as above, create a fake virtio-console 'sink' port when the serial
517     // console is disabled, so the PCI device ID assignments don't move
518     // around
519     add_hvc_sink();
520   }
521 
522   // /dev/hvc2 = serial logging
523   // Serial port for logcat, redirected to a pipe
524   add_hvc_ro(instance.logcat_pipe_name());
525 
526   // /dev/hvc3 = keymaster (C++ implementation)
527   add_hvc(instance.PerInstanceInternalPath("keymaster_fifo_vm"));
528   // /dev/hvc4 = gatekeeper
529   add_hvc(instance.PerInstanceInternalPath("gatekeeper_fifo_vm"));
530   // /dev/hvc5 = bt
531   if (config.enable_host_bluetooth()) {
532     add_hvc(instance.PerInstanceInternalPath("bt_fifo_vm"));
533   } else {
534     add_hvc_sink();
535   }
536 
537   // /dev/hvc6 = gnss
538   // /dev/hvc7 = location
539   if (instance.enable_gnss_grpc_proxy()) {
540     add_hvc(instance.PerInstanceInternalPath("gnsshvc_fifo_vm"));
541     add_hvc(instance.PerInstanceInternalPath("locationhvc_fifo_vm"));
542   } else {
543     for (auto i = 0; i < 2; i++) {
544       add_hvc_sink();
545     }
546   }
547 
548   /* Added one for confirmation UI.
549    *
550    * b/237452165
551    *
552    * Confirmation UI is not supported with QEMU for now. In order
553    * to not conflict with confirmation UI-related configurations used
554    * w/ Crosvm, we should add one generic avc.
555    *
556    * confui_fifo_vm.{in/out} are created along with the streamer process,
557    * which is not created w/ QEMU.
558    */
559   // /dev/hvc8 = confirmationui
560   add_hvc_sink();
561 
562   // /dev/hvc9 = uwb
563   if (config.enable_host_uwb()) {
564     add_hvc(instance.PerInstanceInternalPath("uwb_fifo_vm"));
565   } else {
566     add_hvc_sink();
567   }
568 
569   // /dev/hvc10 = oemlock
570   add_hvc(instance.PerInstanceInternalPath("oemlock_fifo_vm"));
571 
572   // /dev/hvc11 = keymint (Rust implementation)
573   add_hvc(instance.PerInstanceInternalPath("keymint_fifo_vm"));
574 
575   // /dev/hvc12 = nfc
576   if (config.enable_host_nfc()) {
577     add_hvc(instance.PerInstanceInternalPath("nfc_fifo_vm"));
578   } else {
579     add_hvc_sink();
580   }
581 
582   // sensors_fifo_vm.{in/out} are created along with the streamer process,
583   // which is not created w/ QEMU.
584   // /dev/hvc13 = sensors
585   add_hvc_sink();
586 
587   // /dev/hvc14 = MCU CONTROL
588   if (instance.mcu()["control"]["type"].asString() == "serial") {
589     auto path = instance.PerInstanceInternalPath("mcu");
590     path += "/" + instance.mcu()["control"]["path"].asString();
591     add_hvc_serial(path);
592   } else {
593     add_hvc_sink();
594   }
595 
596   // /dev/hvc15 = MCU UART
597   if (instance.mcu()["uart0"]["type"].asString() == "serial") {
598     auto path = instance.PerInstanceInternalPath("mcu");
599     path += "/" + instance.mcu()["uart0"]["path"].asString();
600     add_hvc_serial(path);
601   } else {
602     add_hvc_sink();
603   }
604 
605   auto disk_num = instance.virtual_disk_paths().size();
606 
607   for (auto i = 0; i < VmManager::kMaxDisks - disk_num; i++) {
608     add_hvc_sink();
609   }
610 
611   CF_EXPECT(
612       hvc_num + disk_num == VmManager::kMaxDisks + VmManager::kDefaultNumHvcs,
613       "HVC count (" << hvc_num << ") + disk count (" << disk_num << ") "
614                     << "is not the expected total of "
615                     << VmManager::kMaxDisks + VmManager::kDefaultNumHvcs
616                     << " devices");
617 
618   CF_EXPECT(VmManager::kMaxDisks >= disk_num,
619             "Provided too many disks (" << disk_num << "), maximum "
620                                         << VmManager::kMaxDisks << "supported");
621   auto readonly = instance.protected_vm() ? ",readonly" : "";
622   size_t i = 0;
623   for (const auto& disk : instance.virtual_disk_paths()) {
624     qemu_cmd.AddParameter("-drive");
625     qemu_cmd.AddParameter("file=", disk, ",if=none,id=drive-virtio-disk", i,
626                           ",aio=threads", readonly);
627     qemu_cmd.AddParameter("-device");
628     qemu_cmd.AddParameter(
629 #ifdef __APPLE__
630         "virtio-blk-pci-non-transitional,drive=drive-virtio-disk", i,
631 #else
632         "virtio-blk-pci-non-transitional,scsi=off,drive=drive-virtio-disk", i,
633 #endif
634         ",id=virtio-disk", i, (i == 0 ? ",bootindex=1" : ""));
635     ++i;
636   }
637 
638   if (is_x86 && FileExists(instance.pstore_path())) {
639     // QEMU will assign the NVDIMM (ramoops pstore region) 150000000-1501fffff
640     // As we will pass this to ramoops, define this region first so it is always
641     // located at this address. This is currently x86 only.
642     qemu_cmd.AddParameter("-object");
643     qemu_cmd.AddParameter("memory-backend-file,id=objpmem0,share=on,mem-path=",
644                           instance.pstore_path(), ",size=", pstore_size_bytes);
645 
646     qemu_cmd.AddParameter("-device");
647     qemu_cmd.AddParameter("nvdimm,memdev=objpmem0,id=ramoops");
648   }
649 
650   // QEMU does not implement virtio-pmem-pci for ARM64 or RISC-V yet; restore
651   // this when the device has been added
652   if (is_x86) {
653     if (access_kregistry_size_bytes > 0) {
654       qemu_cmd.AddParameter("-object");
655       qemu_cmd.AddParameter(
656           "memory-backend-file,id=objpmem1,share=on,mem-path=",
657           instance.access_kregistry_path(),
658           ",size=", access_kregistry_size_bytes);
659 
660       qemu_cmd.AddParameter("-device");
661       qemu_cmd.AddParameter(
662           "virtio-pmem-pci,disable-legacy=on,memdev=objpmem1,id=pmem0");
663     }
664     if (hwcomposer_pmem_size_bytes > 0) {
665       qemu_cmd.AddParameter("-object");
666       qemu_cmd.AddParameter(
667           "memory-backend-file,id=objpmem2,share=on,mem-path=",
668           instance.hwcomposer_pmem_path(),
669           ",size=", hwcomposer_pmem_size_bytes);
670 
671       qemu_cmd.AddParameter("-device");
672       qemu_cmd.AddParameter(
673           "virtio-pmem-pci,disable-legacy=on,memdev=objpmem2,id=pmem1");
674     }
675   }
676 
677   qemu_cmd.AddParameter("-object");
678   qemu_cmd.AddParameter("rng-random,id=objrng0,filename=/dev/urandom");
679 
680   qemu_cmd.AddParameter("-device");
681   qemu_cmd.AddParameter("virtio-rng-pci-non-transitional,rng=objrng0,id=rng0,",
682                         "max-bytes=1024,period=2000");
683 
684   qemu_cmd.AddParameter("-device");
685   qemu_cmd.AddParameter("virtio-mouse-pci,disable-legacy=on");
686 
687   qemu_cmd.AddParameter("-device");
688   qemu_cmd.AddParameter("virtio-keyboard-pci,disable-legacy=on");
689 
690   // device padding for unsupported "switches" input
691   qemu_cmd.AddParameter("-device");
692   qemu_cmd.AddParameter("virtio-keyboard-pci,disable-legacy=on");
693 
694   auto vhost_net = instance.vhost_net() ? ",vhost=on" : "";
695 
696   qemu_cmd.AddParameter("-device");
697   qemu_cmd.AddParameter("virtio-balloon-pci-non-transitional,id=balloon0");
698 
699   switch (instance.external_network_mode()) {
700     case ExternalNetworkMode::kTap:
701       qemu_cmd.AddParameter("-netdev");
702       qemu_cmd.AddParameter(
703           "tap,id=hostnet0,ifname=", instance.mobile_tap_name(),
704           ",script=no,downscript=no", vhost_net);
705 
706       qemu_cmd.AddParameter("-netdev");
707       qemu_cmd.AddParameter(
708           "tap,id=hostnet1,ifname=", instance.ethernet_tap_name(),
709           ",script=no,downscript=no", vhost_net);
710 
711       if (!config.virtio_mac80211_hwsim()) {
712         qemu_cmd.AddParameter("-netdev");
713         qemu_cmd.AddParameter(
714             "tap,id=hostnet2,ifname=", instance.wifi_tap_name(),
715             ",script=no,downscript=no", vhost_net);
716       }
717       break;
718     case cuttlefish::ExternalNetworkMode::kSlirp: {
719       const std::string net =
720           fmt::format("{}/{}", instance.ril_ipaddr(), instance.ril_prefixlen());
721       const std::string& host = instance.ril_gateway();
722       qemu_cmd.AddParameter("-netdev");
723       // TODO(schuffelen): `dns` needs to match the first `nameserver` in
724       // `/etc/resolv.conf`. Implement something that generalizes beyond gLinux.
725       qemu_cmd.AddParameter("user,id=hostnet0,net=", net, ",host=", host,
726                             ",dns=127.0.0.1");
727 
728       qemu_cmd.AddParameter("-netdev");
729       qemu_cmd.AddParameter("user,id=hostnet1,net=10.0.1.1/24,dns=8.8.4.4");
730 
731       if (!config.virtio_mac80211_hwsim()) {
732         qemu_cmd.AddParameter("-netdev");
733         qemu_cmd.AddParameter("user,id=hostnet2,net=10.0.2.1/24,dns=1.1.1.1");
734       }
735       break;
736     }
737     default:
738       return CF_ERRF("Unexpected net mode {}",
739                      instance.external_network_mode());
740   }
741 
742   // The ordering of virtio-net devices is important. Make sure any change here
743   // is reflected in ethprime u-boot variable
744   qemu_cmd.AddParameter("-device");
745   qemu_cmd.AddParameter(
746       "virtio-net-pci-non-transitional,netdev=hostnet0,id=net0,mac=",
747       instance.mobile_mac());
748   qemu_cmd.AddParameter("-device");
749   qemu_cmd.AddParameter("virtio-net-pci-non-transitional,netdev=hostnet1,id=net1,mac=",
750                         instance.ethernet_mac());
751   if (!config.virtio_mac80211_hwsim()) {
752     qemu_cmd.AddParameter("-device");
753     qemu_cmd.AddParameter("virtio-net-pci-non-transitional,netdev=hostnet2,id=net2,mac=",
754                           instance.wifi_mac());
755   }
756 
757   if (is_x86 || is_arm) {
758     qemu_cmd.AddParameter("-cpu");
759     qemu_cmd.AddParameter(IsHostCompatible(arch_) ? "host" : "max");
760   }
761 
762   // Explicitly enable the optional extensions of interest, in case the default
763   // behavior changes upstream.
764   if (is_riscv64) {
765     qemu_cmd.AddParameter("-cpu");
766     qemu_cmd.AddParameter("rv64",
767                           ",v=true,elen=64,vlen=128",
768                           ",zba=true,zbb=true,zbs=true");
769   }
770 
771   qemu_cmd.AddParameter("-msg");
772   qemu_cmd.AddParameter("timestamp=on");
773 
774 #ifdef __linux__
775   qemu_cmd.AddParameter("-device");
776   qemu_cmd.AddParameter("vhost-vsock-pci-non-transitional,guest-cid=",
777                         instance.vsock_guest_cid());
778 #endif
779 
780   qemu_cmd.AddParameter("-device");
781   qemu_cmd.AddParameter("AC97,audiodev=audio_none");
782   qemu_cmd.AddParameter("-audiodev");
783   qemu_cmd.AddParameter("driver=none,id=audio_none");
784 
785   qemu_cmd.AddParameter("-device");
786   qemu_cmd.AddParameter("qemu-xhci,id=xhci");
787 
788   qemu_cmd.AddParameter("-L");
789   qemu_cmd.AddParameter(HostQemuBiosPath());
790 
791   if (is_riscv64) {
792     qemu_cmd.AddParameter("-kernel");
793     qemu_cmd.AddParameter(instance.bootloader());
794   } else if (is_arm) {
795     qemu_cmd.AddParameter("-bios");
796     qemu_cmd.AddParameter(instance.bootloader());
797   } else {
798     qemu_cmd.AddParameter("-drive");
799     qemu_cmd.AddParameter("if=pflash,format=raw,readonly=on,file=",
800                           instance.bootloader());
801     qemu_cmd.AddParameter("-drive");
802     qemu_cmd.AddParameter("if=pflash,format=raw,file=", instance.pflash_path());
803   }
804 
805   if (instance.gdb_port() > 0) {
806     qemu_cmd.AddParameter("-S");
807     qemu_cmd.AddParameter("-gdb");
808     qemu_cmd.AddParameter("tcp::", instance.gdb_port());
809   }
810 
811   std::vector<MonitorCommand> commands;
812   commands.emplace_back(std::move(qemu_cmd), true);
813   return commands;
814 }
815 
816 } // namespace vm_manager
817 }  // namespace cuttlefish
818