1# python3
2# Copyright (C) 2019 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16"""Grep warnings messages and output HTML tables or warning counts in CSV.
17
18Default is to output warnings in HTML tables grouped by warning severity.
19Use option --byproject to output tables grouped by source file projects.
20Use option --gencsv to output warning counts in CSV format.
21
22Default input file is build.log, which can be changed with the --log flag.
23"""
24
25# List of important data structures and functions in this script.
26#
27# To parse and keep warning message in the input file:
28#   severity:                classification of message severity
29#   warn_patterns:
30#   warn_patterns[w]['category']     tool that issued the warning, not used now
31#   warn_patterns[w]['description']  table heading
32#   warn_patterns[w]['members']      matched warnings from input
33#   warn_patterns[w]['patterns']     regular expressions to match warnings
34#   warn_patterns[w]['projects'][p]  number of warnings of pattern w in p
35#   warn_patterns[w]['severity']     severity tuple
36#   project_list[p][0]               project name
37#   project_list[p][1]               regular expression to match a project path
38#   project_patterns[p]              re.compile(project_list[p][1])
39#   project_names[p]                 project_list[p][0]
40#   warning_messages     array of each warning message, without source url
41#   warning_links        array of each warning code search link; for 'chrome'
42#   warning_records      array of [idx to warn_patterns,
43#                                  idx to project_names,
44#                                  idx to warning_messages,
45#                                  idx to warning_links]
46#   parse_input_file
47#
48import argparse
49import io
50import multiprocessing
51import os
52import re
53import sys
54
55# pylint:disable=relative-beyond-top-level,no-name-in-module
56# suppress false positive of no-name-in-module warnings
57from . import android_project_list
58from . import chrome_project_list
59from . import cpp_warn_patterns as cpp_patterns
60from . import html_writer
61from . import java_warn_patterns as java_patterns
62from . import make_warn_patterns as make_patterns
63from . import other_warn_patterns as other_patterns
64from . import tidy_warn_patterns as tidy_patterns
65
66
67# Location of this file is used to guess the root of Android source tree.
68THIS_FILE_PATH = 'build/make/tools/warn/warn_common.py'
69
70
71def parse_args(use_google3):
72  """Define and parse the args. Return the parse_args() result."""
73  parser = argparse.ArgumentParser(
74      description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
75  parser.add_argument('--capacitor_path', default='',
76                      help='Save capacitor warning file to the passed absolute'
77                      ' path')
78  # csvpath has a different naming than the above path because historically the
79  # original Android script used csvpath, so other scripts rely on it
80  parser.add_argument('--csvpath', default='',
81                      help='Save CSV warning file to the passed path')
82  parser.add_argument('--gencsv', action='store_true',
83                      help='Generate CSV file with number of various warnings')
84  parser.add_argument('--csvwithdescription', default='',
85                      help="""Save CSV warning file to the passed path this csv
86                            will contain all the warning descriptions""")
87  parser.add_argument('--byproject', action='store_true',
88                      help='Separate warnings in HTML output by project names')
89  parser.add_argument('--url', default='',
90                      help='Root URL of an Android source code tree prefixed '
91                      'before files in warnings')
92  parser.add_argument('--separator', default='?l=',
93                      help='Separator between the end of a URL and the line '
94                      'number argument. e.g. #')
95  parser.add_argument('--processes', default=multiprocessing.cpu_count(),
96                      type=int,
97                      help='Number of parallel processes to process warnings')
98  # Old Android build scripts call warn.py without --platform,
99  # so the default platform is set to 'android'.
100  parser.add_argument('--platform', default='android',
101                      choices=['chrome', 'android'],
102                      help='Platform of the build log')
103  # Old Android build scripts call warn.py with only a build.log file path.
104  parser.add_argument('--log', help='Path to build log file')
105  parser.add_argument(dest='buildlog', metavar='build.log',
106                      default='build.log', nargs='?',
107                      help='Path to build.log file')
108  flags = parser.parse_args()
109  if not flags.log:
110    flags.log = flags.buildlog
111  if not use_google3 and not os.path.exists(flags.log):
112    sys.exit('Cannot find log file: ' + flags.log)
113  return flags
114
115
116def get_project_names(project_list):
117  """Get project_names from project_list."""
118  return [p[0] for p in project_list]
119
120
121def find_project_index(line, project_patterns):
122  """Return the index to the project pattern array."""
123  for idx, pattern in enumerate(project_patterns):
124    if pattern.match(line):
125      return idx
126  return -1
127
128
129def classify_one_warning(warning, link, results, project_patterns,
130                         warn_patterns):
131  """Classify one warning line."""
132  for idx, pattern in enumerate(warn_patterns):
133    for cpat in pattern['compiled_patterns']:
134      if cpat.match(warning):
135        project_idx = find_project_index(warning, project_patterns)
136        results.append([warning, link, idx, project_idx])
137        return
138  # If we end up here, there was a problem parsing the log
139  # probably caused by 'make -j' mixing the output from
140  # 2 or more concurrent compiles
141
142
143def remove_prefix(src, sub):
144  """Remove everything before last occurrence of substring sub in string src."""
145  if sub in src:
146    inc_sub = src.rfind(sub)
147    return src[inc_sub:]
148  return src
149
150
151# TODO(emmavukelj): Don't have any generate_*_cs_link functions call
152# normalize_path a second time (the first time being in parse_input_file)
153def generate_cs_link(warning_line, flags, android_root=None):
154  """Try to add code search HTTP URL prefix."""
155  if flags.platform == 'chrome':
156    return generate_chrome_cs_link(warning_line, flags)
157  if flags.platform == 'android':
158    return generate_android_cs_link(warning_line, flags, android_root)
159  return 'https://cs.corp.google.com/'
160
161
162def generate_android_cs_link(warning_line, flags, android_root):
163  """Generate the code search link for a warning line in Android."""
164  # max_splits=2 -> only 3 items
165  raw_path, line_number_str, _ = warning_line.split(':', 2)
166  normalized_path = normalize_path(raw_path, flags, android_root)
167  if not flags.url:
168    return normalized_path
169  link_path = flags.url + '/' + normalized_path
170  if line_number_str.isdigit():
171    link_path += flags.separator + line_number_str
172  return link_path
173
174
175def generate_chrome_cs_link(warning_line, flags):
176  """Generate the code search link for a warning line in Chrome."""
177  split_line = warning_line.split(':')
178  raw_path = split_line[0]
179  normalized_path = normalize_path(raw_path, flags)
180  link_base = 'https://cs.chromium.org/'
181  link_add = 'chromium'
182  link_path = None
183
184  # Basically just going through a few specific directory cases and specifying
185  # the proper behavior for that case. This list of cases was accumulated
186  # through trial and error manually going through the warnings.
187  #
188  # This code pattern of using case-specific "if"s instead of "elif"s looks
189  # possibly accidental and mistaken but it is intentional because some paths
190  # fall under several cases (e.g. third_party/lib/nghttp2_frame.c) and for
191  # those we want the most specific case to be applied. If there is reliable
192  # knowledge of exactly where these occur, this could be changed to "elif"s
193  # but there is no reliable set of paths falling under multiple cases at the
194  # moment.
195  if '/src/third_party' in raw_path:
196    link_path = remove_prefix(raw_path, '/src/third_party/')
197  if '/chrome_root/src_internal/' in raw_path:
198    link_path = remove_prefix(raw_path, '/chrome_root/src_internal/')
199    link_path = link_path[len('/chrome_root'):]  # remove chrome_root
200  if '/chrome_root/src/' in raw_path:
201    link_path = remove_prefix(raw_path, '/chrome_root/src/')
202    link_path = link_path[len('/chrome_root'):]  # remove chrome_root
203  if '/libassistant/' in raw_path:
204    link_add = 'eureka_internal/chromium/src'
205    link_base = 'https://cs.corp.google.com/'  # internal data
206    link_path = remove_prefix(normalized_path, '/libassistant/')
207  if raw_path.startswith('gen/'):
208    link_path = '/src/out/Debug/gen/' + normalized_path
209  if '/gen/' in raw_path:
210    return '%s?q=file:%s' % (link_base, remove_prefix(normalized_path, '/gen/'))
211
212  if not link_path and (raw_path.startswith('src/') or
213                        raw_path.startswith('src_internal/')):
214    link_path = '/%s' % raw_path
215
216  if not link_path:  # can't find specific link, send a query
217    return '%s?q=file:%s' % (link_base, normalized_path)
218
219  line_number = int(split_line[1])
220  link = '%s%s%s?l=%d' % (link_base, link_add, link_path, line_number)
221  return link
222
223
224def find_this_file_and_android_root(path):
225  """Return android source root path if this file is found."""
226  parts = path.split('/')
227  for idx in reversed(range(2, len(parts))):
228    root_path = '/'.join(parts[:idx])
229    # Android root directory should contain this script.
230    if os.path.exists(root_path + '/' + THIS_FILE_PATH):
231      return root_path
232  return ''
233
234
235def find_android_root_top_dirs(root_dir):
236  """Return a list of directories under the root_dir, if it exists."""
237  # Root directory should contain at least build/make and build/soong.
238  if (not os.path.isdir(root_dir + '/build/make') or
239      not os.path.isdir(root_dir + '/build/soong')):
240    return None
241  return list(filter(lambda d: os.path.isdir(root_dir + '/' + d),
242                     os.listdir(root_dir)))
243
244
245def find_android_root(buildlog):
246  """Guess android source root from common prefix of file paths."""
247  # Use the longest common prefix of the absolute file paths
248  # of the first 10000 warning messages as the android_root.
249  warning_lines = []
250  warning_pattern = re.compile('^/[^ ]*/[^ ]*: warning: .*')
251  count = 0
252  for line in buildlog:
253    # We want to find android_root of a local build machine.
254    # Do not use RBE warning lines, which has '/b/f/w/' path prefix.
255    # Do not use /tmp/ file warnings.
256    if ('/b/f/w' not in line and not line.startswith('/tmp/') and
257        warning_pattern.match(line)):
258      warning_lines.append(line)
259      count += 1
260      if count > 9999:
261        break
262      # Try to find warn.py and use its location to find
263      # the source tree root.
264      if count < 100:
265        path = os.path.normpath(re.sub(':.*$', '', line))
266        android_root = find_this_file_and_android_root(path)
267        if android_root:
268          return android_root, find_android_root_top_dirs(android_root)
269  # Do not use common prefix of a small number of paths.
270  android_root = ''
271  if count > 10:
272    # pytype: disable=wrong-arg-types
273    root_path = os.path.commonprefix(warning_lines)
274    # pytype: enable=wrong-arg-types
275    if len(root_path) > 2 and root_path[len(root_path) - 1] == '/':
276      android_root = root_path[:-1]
277  if android_root and os.path.isdir(android_root):
278    return android_root, find_android_root_top_dirs(android_root)
279  # When the build.log file is moved to a different machine where
280  # android_root is not found, use the location of this script
281  # to find the android source tree sub directories.
282  if __file__.endswith('/' + THIS_FILE_PATH):
283    script_root = __file__.replace('/' + THIS_FILE_PATH, '')
284    return android_root, find_android_root_top_dirs(script_root)
285  return android_root, None
286
287
288def remove_android_root_prefix(path, android_root):
289  """Remove android_root prefix from path if it is found."""
290  if path.startswith(android_root):
291    return path[1 + len(android_root):]
292  return path
293
294
295def normalize_path(path, flags, android_root=None):
296  """Normalize file path relative to src/ or src-internal/ directory."""
297  path = os.path.normpath(path)
298
299  if flags.platform == 'android':
300    if android_root:
301      return remove_android_root_prefix(path, android_root)
302    return path
303
304  # Remove known prefix of root path and normalize the suffix.
305  idx = path.find('chrome_root/')
306  if idx >= 0:
307    # remove chrome_root/, we want path relative to that
308    return path[idx + len('chrome_root/'):]
309  return path
310
311
312def normalize_warning_line(line, flags, android_root=None):
313  """Normalize file path relative to src directory in a warning line."""
314  line = re.sub(u'[\u2018\u2019]', '\'', line)
315  # replace non-ASCII chars to spaces
316  line = re.sub(u'[^\x00-\x7f]', ' ', line)
317  line = line.strip()
318  first_column = line.find(':')
319  return normalize_path(line[:first_column], flags,
320                        android_root) + line[first_column:]
321
322
323def parse_input_file_chrome(infile, flags):
324  """Parse Chrome input file, collect parameters and warning lines."""
325  platform_version = 'unknown'
326  board_name = 'unknown'
327  architecture = 'unknown'
328
329  # only handle warning lines of format 'file_path:line_no:col_no: warning: ...'
330  # Bug: http://198657613, This might need change to handle RBE output.
331  chrome_warning_pattern = r'^[^ ]*/[^ ]*:[0-9]+:[0-9]+: warning: .*'
332
333  warning_pattern = re.compile(chrome_warning_pattern)
334
335  # Collect all unique warning lines
336  unique_warnings = dict()
337  for line in infile:
338    if warning_pattern.match(line):
339      normalized_line = normalize_warning_line(line, flags)
340      if normalized_line not in unique_warnings:
341        unique_warnings[normalized_line] = generate_cs_link(line, flags)
342    elif (platform_version == 'unknown' or board_name == 'unknown' or
343          architecture == 'unknown'):
344      result = re.match(r'.+Package:.+chromeos-base/chromeos-chrome-', line)
345      if result is not None:
346        platform_version = 'R' + line.split('chrome-')[1].split('_')[0]
347        continue
348      result = re.match(r'.+Source\sunpacked\sin\s(.+)', line)
349      if result is not None:
350        board_name = result.group(1).split('/')[2]
351        continue
352      result = re.match(r'.+USE:\s*([^\s]*).*', line)
353      if result is not None:
354        architecture = result.group(1)
355        continue
356
357  header_str = '%s - %s - %s' % (platform_version, board_name, architecture)
358  return unique_warnings, header_str
359
360
361def add_normalized_line_to_warnings(line, flags, android_root, unique_warnings):
362  """Parse/normalize path, updating warning line and add to warnings dict."""
363  normalized_line = normalize_warning_line(line, flags, android_root)
364  if normalized_line not in unique_warnings:
365    unique_warnings[normalized_line] = generate_cs_link(line, flags,
366                                                        android_root)
367  return unique_warnings
368
369
370def parse_input_file_android(infile, flags):
371  """Parse Android input file, collect parameters and warning lines."""
372  # pylint:disable=too-many-locals,too-many-branches
373  platform_version = 'unknown'
374  target_product = 'unknown'
375  target_variant = 'unknown'
376  build_id = 'unknown'
377  android_root, root_top_dirs = find_android_root(infile)
378  infile.seek(0)
379
380  # rustc warning messages have two lines that should be combined:
381  #     warning: description
382  #        --> file_path:line_number:column_number
383  # Some warning messages have no file name:
384  #     warning: macro replacement list ... [bugprone-macro-parentheses]
385  # Some makefile warning messages have no line number:
386  #     some/path/file.mk: warning: description
387  # C/C++ compiler warning messages have line and column numbers:
388  #     some/path/file.c:line_number:column_number: warning: description
389  warning_pattern = re.compile('(^[^ ]*/[^ ]*: warning: .*)|(^warning: .*)')
390  rustc_file_position = re.compile('^[ ]+--> [^ ]*/[^ ]*:[0-9]+:[0-9]+')
391
392  # If RBE was used, try to reclaim some warning lines (from stdout)
393  # that contain leading characters from stderr.
394  # The leading characters can be any character, including digits and spaces.
395
396  # If a warning line's source file path contains the special RBE prefix
397  # /b/f/w/, we can remove all leading chars up to and including the "/b/f/w/".
398  bfw_warning_pattern = re.compile('.*/b/f/w/([^ ]*: warning: .*)')
399
400  # When android_root is known and available, we find its top directories
401  # and remove all leading chars before a top directory name.
402  # We assume that the leading chars from stderr do not contain "/".
403  # For example,
404  #   10external/...
405  #   12 warningsexternal/...
406  #   413 warningexternal/...
407  #   5 warnings generatedexternal/...
408  #   Suppressed 1000 warnings (packages/modules/...
409  if root_top_dirs:
410    extra_warning_pattern = re.compile(
411        '^.[^/]*((' + '|'.join(root_top_dirs) +
412        ')/[^ ]*: warning: .*)')
413  else:
414    extra_warning_pattern = re.compile('^[^/]* ([^ /]*/[^ ]*: warning: .*)')
415
416  # Collect all unique warning lines
417  unique_warnings = dict()
418  checked_warning_lines = dict()
419  line_counter = 0
420  prev_warning = ''
421  for line in infile:
422    line_counter += 1
423    if prev_warning:
424      if rustc_file_position.match(line):
425        # must be a rustc warning, combine 2 lines into one warning
426        line = line.strip().replace('--> ', '') + ': ' + prev_warning
427        unique_warnings = add_normalized_line_to_warnings(
428            line, flags, android_root, unique_warnings)
429        prev_warning = ''
430        continue
431      # add prev_warning, and then process the current line
432      prev_warning = 'unknown_source_file: ' + prev_warning
433      unique_warnings = add_normalized_line_to_warnings(
434          prev_warning, flags, android_root, unique_warnings)
435      prev_warning = ''
436
437    # re.match is slow, with several warning line patterns and
438    # long input lines like "TIMEOUT: ...".
439    # We save significant time by skipping non-warning lines.
440    # But do not skip the first 100 lines, because we want to
441    # catch build variables.
442    if line_counter > 100 and line.find('warning: ') < 0:
443      continue
444
445    # A large clean build output can contain up to 90% of duplicated
446    # "warning:" lines. If we can skip them quickly, we can
447    # speed up this for-loop 3X to 5X.
448    if line in checked_warning_lines:
449      continue
450    checked_warning_lines[line] = True
451
452    # Clean up extra prefix that could be introduced when RBE was used.
453    if '/b/f/w/' in line:
454      result = bfw_warning_pattern.search(line)
455    else:
456      result = extra_warning_pattern.search(line)
457    if result is not None:
458      line = result.group(1)
459
460    if warning_pattern.match(line):
461      if line.startswith('warning: '):
462        # save this line and combine it with the next line
463        prev_warning = line
464      else:
465        unique_warnings = add_normalized_line_to_warnings(
466            line, flags, android_root, unique_warnings)
467      continue
468
469    if line_counter < 100:
470      # save a little bit of time by only doing this for the first few lines
471      result = re.search('(?<=^PLATFORM_VERSION=).*', line)
472      if result is not None:
473        platform_version = result.group(0)
474        continue
475      result = re.search('(?<=^TARGET_PRODUCT=).*', line)
476      if result is not None:
477        target_product = result.group(0)
478        continue
479      result = re.search('(?<=^TARGET_BUILD_VARIANT=).*', line)
480      if result is not None:
481        target_variant = result.group(0)
482        continue
483      result = re.search('(?<=^BUILD_ID=).*', line)
484      if result is not None:
485        build_id = result.group(0)
486        continue
487
488  if android_root:
489    new_unique_warnings = dict()
490    for warning_line in unique_warnings:
491      normalized_line = normalize_warning_line(warning_line, flags,
492                                               android_root)
493      new_unique_warnings[normalized_line] = generate_android_cs_link(
494          warning_line, flags, android_root)
495    unique_warnings = new_unique_warnings
496
497  header_str = '%s - %s - %s (%s)' % (
498      platform_version, target_product, target_variant, build_id)
499  return unique_warnings, header_str
500
501
502def parse_input_file(infile, flags):
503  """Parse one input file for chrome or android."""
504  if flags.platform == 'chrome':
505    return parse_input_file_chrome(infile, flags)
506  if flags.platform == 'android':
507    return parse_input_file_android(infile, flags)
508  raise RuntimeError('parse_input_file not defined for platform %s' %
509                     flags.platform)
510
511
512def parse_compiler_output(compiler_output):
513  """Parse compiler output for relevant info."""
514  split_output = compiler_output.split(':', 3)  # 3 = max splits
515  file_path = split_output[0]
516  line_number = int(split_output[1])
517  col_number = int(split_output[2].split(' ')[0])
518  warning_message = split_output[3]
519  return file_path, line_number, col_number, warning_message
520
521
522def get_warn_patterns(platform):
523  """Get and initialize warn_patterns."""
524  warn_patterns = []
525  if platform == 'chrome':
526    warn_patterns = cpp_patterns.warn_patterns
527  elif platform == 'android':
528    warn_patterns = (make_patterns.warn_patterns + cpp_patterns.warn_patterns +
529                     java_patterns.warn_patterns + tidy_patterns.warn_patterns +
530                     other_patterns.warn_patterns)
531  else:
532    raise Exception('platform name %s is not valid' % platform)
533  for pattern in warn_patterns:
534    pattern['members'] = []
535    # Each warning pattern has a 'projects' dictionary, that
536    # maps a project name to number of warnings in that project.
537    pattern['projects'] = {}
538  return warn_patterns
539
540
541def get_project_list(platform):
542  """Return project list for appropriate platform."""
543  if platform == 'chrome':
544    return chrome_project_list.project_list
545  if platform == 'android':
546    return android_project_list.project_list
547  raise Exception('platform name %s is not valid' % platform)
548
549
550def parallel_classify_warnings(warning_data, args, project_names,
551                               project_patterns, warn_patterns,
552                               use_google3, create_launch_subprocs_fn,
553                               classify_warnings_fn):
554  """Classify all warning lines with num_cpu parallel processes."""
555  # pylint:disable=too-many-arguments,too-many-locals
556  num_cpu = args.processes
557  group_results = []
558
559  if num_cpu > 1:
560    # set up parallel processing for this...
561    warning_groups = [[] for _ in range(num_cpu)]
562    i = 0
563    for warning, link in warning_data.items():
564      warning_groups[i].append((warning, link))
565      i = (i + 1) % num_cpu
566    arg_groups = [[] for _ in range(num_cpu)]
567    for i, group in enumerate(warning_groups):
568      arg_groups[i] = [{
569          'group': group,
570          'project_patterns': project_patterns,
571          'warn_patterns': warn_patterns,
572          'num_processes': num_cpu
573      }]
574
575    group_results = create_launch_subprocs_fn(num_cpu,
576                                              classify_warnings_fn,
577                                              arg_groups,
578                                              group_results)
579  else:
580    group_results = []
581    for warning, link in warning_data.items():
582      classify_one_warning(warning, link, group_results,
583                           project_patterns, warn_patterns)
584    group_results = [group_results]
585
586  warning_messages = []
587  warning_links = []
588  warning_records = []
589  if use_google3:
590    group_results = [group_results]
591  for group_result in group_results:
592    for result in group_result:
593      for line, link, pattern_idx, project_idx in result:
594        pattern = warn_patterns[pattern_idx]
595        pattern['members'].append(line)
596        message_idx = len(warning_messages)
597        warning_messages.append(line)
598        link_idx = len(warning_links)
599        warning_links.append(link)
600        warning_records.append([pattern_idx, project_idx, message_idx,
601                                link_idx])
602        pname = '???' if project_idx < 0 else project_names[project_idx]
603        # Count warnings by project.
604        if pname in pattern['projects']:
605          pattern['projects'][pname] += 1
606        else:
607          pattern['projects'][pname] = 1
608  return warning_messages, warning_links, warning_records
609
610
611def process_log(logfile, flags, project_names, project_patterns, warn_patterns,
612                html_path, use_google3, create_launch_subprocs_fn,
613                classify_warnings_fn, logfile_object):
614  # pylint does not recognize g-doc-*
615  # pylint: disable=bad-option-value,g-doc-args
616  # pylint: disable=bad-option-value,g-doc-return-or-yield
617  # pylint: disable=too-many-arguments,too-many-locals
618  """Function that handles processing of a log.
619
620  This is isolated into its own function (rather than just taking place in main)
621  so that it can be used by both warn.py and the borg job process_gs_logs.py, to
622  avoid duplication of code.
623  Note that if the arguments to this function change, process_gs_logs.py must
624  be updated accordingly.
625  """
626  if logfile_object is None:
627    with io.open(logfile, encoding='utf-8') as log:
628      warning_lines_and_links, header_str = parse_input_file(log, flags)
629  else:
630    warning_lines_and_links, header_str = parse_input_file(
631        logfile_object, flags)
632  warning_messages, warning_links, warning_records = parallel_classify_warnings(
633      warning_lines_and_links, flags, project_names, project_patterns,
634      warn_patterns, use_google3, create_launch_subprocs_fn,
635      classify_warnings_fn)
636
637  html_writer.write_html(flags, project_names, warn_patterns, html_path,
638                         warning_messages, warning_links, warning_records,
639                         header_str)
640
641  return warning_messages, warning_links, warning_records, header_str
642
643
644def common_main(use_google3, create_launch_subprocs_fn, classify_warnings_fn,
645                logfile_object=None):
646  """Shared main function for Google3 and non-Google3 versions of warn.py."""
647  flags = parse_args(use_google3)
648  warn_patterns = get_warn_patterns(flags.platform)
649  project_list = get_project_list(flags.platform)
650
651  project_names = get_project_names(project_list)
652  project_patterns = [re.compile(p[1]) for p in project_list]
653
654  # html_path=None because we output html below if not outputting CSV
655  warning_messages, warning_links, warning_records, header_str = process_log(
656      logfile=flags.log, flags=flags, project_names=project_names,
657      project_patterns=project_patterns, warn_patterns=warn_patterns,
658      html_path=None, use_google3=use_google3,
659      create_launch_subprocs_fn=create_launch_subprocs_fn,
660      classify_warnings_fn=classify_warnings_fn,
661      logfile_object=logfile_object)
662
663  html_writer.write_out_csv(flags, warn_patterns, warning_messages,
664                            warning_links, warning_records, header_str,
665                            project_names)
666
667  # Return these values, so that caller can use them, if desired.
668  return flags, warning_messages, warning_records, warn_patterns
669