1 //
2 // Copyright (C) 2012 The Android Open Source Project
3 //
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
7 //
8 //      http://www.apache.org/licenses/LICENSE-2.0
9 //
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
15 //
16 
17 #include "update_engine/payload_generator/delta_diff_generator.h"
18 
19 #include <errno.h>
20 #include <fcntl.h>
21 #include <inttypes.h>
22 #include <sys/stat.h>
23 #include <sys/types.h>
24 
25 #include <algorithm>
26 #include <memory>
27 #include <string>
28 #include <utility>
29 #include <vector>
30 
31 #include <base/logging.h>
32 #include <base/threading/simple_thread.h>
33 
34 #include "update_engine/common/utils.h"
35 #include "update_engine/payload_consumer/delta_performer.h"
36 #include "update_engine/payload_consumer/file_descriptor.h"
37 #include "update_engine/payload_consumer/payload_constants.h"
38 #include "update_engine/payload_generator/ab_generator.h"
39 #include "update_engine/payload_generator/annotated_operation.h"
40 #include "update_engine/payload_generator/blob_file_writer.h"
41 #include "update_engine/payload_generator/cow_size_estimator.h"
42 #include "update_engine/payload_generator/delta_diff_utils.h"
43 #include "update_engine/payload_generator/full_update_generator.h"
44 #include "update_engine/payload_generator/merge_sequence_generator.h"
45 #include "update_engine/payload_generator/payload_file.h"
46 #include "update_engine/update_metadata.pb.h"
47 
48 using std::string;
49 using std::unique_ptr;
50 using std::vector;
51 
52 namespace chromeos_update_engine {
53 
54 // bytes
55 const size_t kRootFSPartitionSize = static_cast<size_t>(2) * 1024 * 1024 * 1024;
56 
57 class PartitionProcessor : public base::DelegateSimpleThread::Delegate {
IsDynamicPartition(const std::string & partition_name)58   bool IsDynamicPartition(const std::string& partition_name) {
59     for (const auto& group :
60          config_.target.dynamic_partition_metadata->groups()) {
61       const auto& names = group.partition_names();
62       if (std::find(names.begin(), names.end(), partition_name) !=
63           names.end()) {
64         return true;
65       }
66     }
67     return false;
68   }
69 
70  public:
PartitionProcessor(const PayloadGenerationConfig & config,const PartitionConfig & old_part,const PartitionConfig & new_part,BlobFileWriter * file_writer,std::vector<AnnotatedOperation> * aops,std::vector<CowMergeOperation> * cow_merge_sequence,android::snapshot::CowSizeInfo * cow_info,std::unique_ptr<chromeos_update_engine::OperationsGenerator> strategy)71   explicit PartitionProcessor(
72       const PayloadGenerationConfig& config,
73       const PartitionConfig& old_part,
74       const PartitionConfig& new_part,
75       BlobFileWriter* file_writer,
76       std::vector<AnnotatedOperation>* aops,
77       std::vector<CowMergeOperation>* cow_merge_sequence,
78       android::snapshot::CowSizeInfo* cow_info,
79       std::unique_ptr<chromeos_update_engine::OperationsGenerator> strategy)
80       : config_(config),
81         old_part_(old_part),
82         new_part_(new_part),
83         file_writer_(file_writer),
84         aops_(aops),
85         cow_merge_sequence_(cow_merge_sequence),
86         cow_info_(cow_info),
87         strategy_(std::move(strategy)) {}
88   PartitionProcessor(PartitionProcessor&&) noexcept = default;
89 
Run()90   void Run() override {
91     LOG(INFO) << "Started an async task to process partition "
92               << new_part_.name;
93     bool success = strategy_->GenerateOperations(
94         config_, old_part_, new_part_, file_writer_, aops_);
95     if (!success) {
96       // ABORT the entire process, so that developer can look
97       // at recent logs and diagnose what happened
98       LOG(FATAL) << "GenerateOperations(" << old_part_.name << ", "
99                  << new_part_.name << ") failed";
100     }
101 
102     bool snapshot_enabled =
103         config_.target.dynamic_partition_metadata &&
104         config_.target.dynamic_partition_metadata->snapshot_enabled();
105     if (!snapshot_enabled || !IsDynamicPartition(new_part_.name)) {
106       return;
107     }
108     // Skip cow size estimation if VABC isn't enabled
109     if (!config_.target.dynamic_partition_metadata->vabc_enabled()) {
110       return;
111     }
112     if (!old_part_.path.empty()) {
113       auto generator = MergeSequenceGenerator::Create(*aops_, new_part_.name);
114       if (!generator || !generator->Generate(cow_merge_sequence_)) {
115         LOG(FATAL) << "Failed to generate merge sequence";
116       }
117     }
118 
119     LOG(INFO) << "Estimating COW size for partition: " << new_part_.name;
120     // Need the contents of source/target image bytes when doing
121     // dry run.
122     auto target_fd = std::make_unique<EintrSafeFileDescriptor>();
123     target_fd->Open(new_part_.path.c_str(), O_RDONLY);
124 
125     google::protobuf::RepeatedPtrField<InstallOperation> operations;
126 
127     for (const AnnotatedOperation& aop : *aops_) {
128       *operations.Add() = aop.op;
129     }
130 
131     FileDescriptorPtr source_fd = std::make_shared<EintrSafeFileDescriptor>();
132     source_fd->Open(old_part_.path.c_str(), O_RDONLY);
133 
134     *cow_info_ = EstimateCowSizeInfo(
135         std::move(source_fd),
136         std::move(target_fd),
137         std::move(operations),
138         {cow_merge_sequence_->begin(), cow_merge_sequence_->end()},
139         config_.block_size,
140         config_.target.dynamic_partition_metadata->vabc_compression_param(),
141         new_part_.size,
142         old_part_.size,
143         config_.enable_vabc_xor,
144         config_.target.dynamic_partition_metadata->cow_version(),
145         config_.target.dynamic_partition_metadata->compression_factor());
146 
147     // add a 1% overhead to our estimation
148     cow_info_->cow_size = cow_info_->cow_size * 1.01;
149     if (config_.target.dynamic_partition_metadata->cow_version() >= 3) {
150       cow_info_->op_count_max = std::max(int(cow_info_->op_count_max), 25);
151     }
152     // ops buffer size == 0 for v2 version of cow format
153     LOG(INFO) << "Estimated COW size for partition: " << new_part_.name << " "
154               << cow_info_->cow_size
155               << " ops buffer size: " << cow_info_->op_count_max;
156   }
157 
158  private:
159   const PayloadGenerationConfig& config_;
160   const PartitionConfig& old_part_;
161   const PartitionConfig& new_part_;
162   BlobFileWriter* file_writer_;
163   std::vector<AnnotatedOperation>* aops_;
164   std::vector<CowMergeOperation>* cow_merge_sequence_;
165   android::snapshot::CowSizeInfo* cow_info_;
166   std::unique_ptr<chromeos_update_engine::OperationsGenerator> strategy_;
167   DISALLOW_COPY_AND_ASSIGN(PartitionProcessor);
168 };
169 
GenerateUpdatePayloadFile(const PayloadGenerationConfig & config,const string & output_path,const string & private_key_path,uint64_t * metadata_size)170 bool GenerateUpdatePayloadFile(const PayloadGenerationConfig& config,
171                                const string& output_path,
172                                const string& private_key_path,
173                                uint64_t* metadata_size) {
174   if (!config.version.Validate()) {
175     LOG(ERROR) << "Unsupported major.minor version: " << config.version.major
176                << "." << config.version.minor;
177     return false;
178   }
179 
180   // Create empty payload file object.
181   PayloadFile payload;
182   TEST_AND_RETURN_FALSE(payload.Init(config));
183 
184   ScopedTempFile data_file("CrAU_temp_data.XXXXXX", true);
185   {
186     off_t data_file_size = 0;
187     BlobFileWriter blob_file(data_file.fd(), &data_file_size);
188     if (config.is_delta) {
189       TEST_AND_RETURN_FALSE(config.source.partitions.size() ==
190                             config.target.partitions.size());
191     }
192     PartitionConfig empty_part("");
193     std::vector<std::vector<AnnotatedOperation>> all_aops;
194     all_aops.resize(config.target.partitions.size());
195 
196     std::vector<std::vector<CowMergeOperation>> all_merge_sequences;
197     all_merge_sequences.resize(config.target.partitions.size());
198 
199     std::vector<android::snapshot::CowSizeInfo> all_cow_info(
200         config.target.partitions.size());
201 
202     std::vector<PartitionProcessor> partition_tasks{};
203     auto thread_count = std::min<int>(diff_utils::GetMaxThreads(),
204                                       config.target.partitions.size());
205     base::DelegateSimpleThreadPool thread_pool{"partition-thread-pool",
206                                                thread_count};
207     for (size_t i = 0; i < config.target.partitions.size(); i++) {
208       const PartitionConfig& old_part =
209           config.is_delta ? config.source.partitions[i] : empty_part;
210       const PartitionConfig& new_part = config.target.partitions[i];
211       LOG(INFO) << "Partition name: " << new_part.name;
212       LOG(INFO) << "Partition size: " << new_part.size;
213       LOG(INFO) << "Block count: " << new_part.size / config.block_size;
214 
215       // Select payload generation strategy based on the config.
216       unique_ptr<OperationsGenerator> strategy;
217       if (!old_part.path.empty()) {
218         // Delta update.
219         LOG(INFO) << "Using generator ABGenerator() for partition "
220                   << new_part.name;
221         strategy.reset(new ABGenerator());
222       } else {
223         LOG(INFO) << "Using generator FullUpdateGenerator() for partition "
224                   << new_part.name;
225         strategy.reset(new FullUpdateGenerator());
226       }
227 
228       // Generate the operations using the strategy we selected above.
229       partition_tasks.push_back(PartitionProcessor(config,
230                                                    old_part,
231                                                    new_part,
232                                                    &blob_file,
233                                                    &all_aops[i],
234                                                    &all_merge_sequences[i],
235                                                    &all_cow_info[i],
236                                                    std::move(strategy)));
237     }
238     thread_pool.Start();
239     for (auto& processor : partition_tasks) {
240       thread_pool.AddWork(&processor);
241     }
242     thread_pool.JoinAll();
243 
244     for (size_t i = 0; i < config.target.partitions.size(); i++) {
245       const PartitionConfig& old_part =
246           config.is_delta ? config.source.partitions[i] : empty_part;
247       const PartitionConfig& new_part = config.target.partitions[i];
248       TEST_AND_RETURN_FALSE(
249           payload.AddPartition(old_part,
250                                new_part,
251                                std::move(all_aops[i]),
252                                std::move(all_merge_sequences[i]),
253                                all_cow_info[i]));
254     }
255   }
256   data_file.CloseFd();
257 
258   LOG(INFO) << "Writing payload file...";
259   // Write payload file to disk.
260   TEST_AND_RETURN_FALSE(payload.WritePayload(
261       output_path, data_file.path(), private_key_path, metadata_size));
262 
263   LOG(INFO) << "All done. Successfully created delta file with "
264             << "metadata size = " << *metadata_size;
265   return true;
266 }
267 
268 };  // namespace chromeos_update_engine
269