• Home
  • History
  • Annotate
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 """Provides helper functions for fetching artifacts."""
2 
3 import io
4 import os
5 import re
6 import sys
7 import sysconfig
8 import time
9 
10 # This is a workaround to put '/usr/lib/python3.X' ahead of googleapiclient
11 # Using embedded_launcher won't work since py3-cmd doesn't contain _ssl module.
12 if sys.version_info.major == 3:
13   sys.path.insert(0, os.path.dirname(sysconfig.get_paths()['purelib']))
14 
15 # pylint: disable=import-error,g-bad-import-order,g-import-not-at-top
16 import apiclient
17 from googleapiclient.discovery import build
18 from googleapiclient.errors import HttpError
19 
20 import httplib2
21 from oauth2client.service_account import ServiceAccountCredentials
22 
23 _SCOPE_URL = 'https://www.googleapis.com/auth/androidbuild.internal'
24 _DEF_JSON_KEYFILE = '.config/gcloud/application_default_credentials.json'
25 
26 
27 # 20 MB default chunk size -- used in Buildbot
28 _DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024
29 
30 # HTTP errors -- used in Builbot
31 _DEFAULT_MASKED_ERRORS = [404]
32 _DEFAULT_RETRIED_ERRORS = [503]
33 _DEFAULT_RETRIES = 10
34 
35 
36 def _create_http_from_p12(robot_credentials_file, robot_username):
37   """Creates a credentialed HTTP object for requests.
38 
39   Args:
40     robot_credentials_file: The path to the robot credentials file.
41     robot_username: A string containing the username of the robot account.
42 
43   Returns:
44     An authorized httplib2.Http object.
45   """
46   try:
47     credentials = ServiceAccountCredentials.from_p12_keyfile(
48         service_account_email=robot_username,
49         filename=robot_credentials_file,
50         scopes=_SCOPE_URL)
51   except AttributeError:
52     raise ValueError('Machine lacks openssl or pycrypto support')
53   http = httplib2.Http()
54   return credentials.authorize(http)
55 
56 
57 def _simple_execute(http_request,
58                     masked_errors=None,
59                     retried_errors=None,
60                     retry_delay_seconds=5,
61                     max_tries=_DEFAULT_RETRIES):
62   """Execute http request and return None on specified errors.
63 
64   Args:
65     http_request: the apiclient provided http request
66     masked_errors: list of errors to return None on
67     retried_errors: list of erros to retry the request on
68     retry_delay_seconds: how many seconds to sleep before retrying
69     max_tries: maximum number of attmpts to make request
70 
71   Returns:
72     The result on success or None on masked errors.
73   """
74   if not masked_errors:
75     masked_errors = _DEFAULT_MASKED_ERRORS
76   if not retried_errors:
77     retried_errors = _DEFAULT_RETRIED_ERRORS
78 
79   last_error = None
80   for _ in range(max_tries):
81     try:
82       return http_request.execute()
83     except HttpError as e:
84       last_error = e
85       if e.resp.status in masked_errors:
86         return None
87       elif e.resp.status in retried_errors:
88         time.sleep(retry_delay_seconds)
89       else:
90         # Server Error is server error
91         raise e
92 
93   # We've gone through the max_retries, raise the last error
94   raise last_error  # pylint: disable=raising-bad-type
95 
96 
97 def create_client(http):
98   """Creates an Android build api client from an authorized http object.
99 
100   Args:
101      http: An authorized httplib2.Http object.
102 
103   Returns:
104     An authorized android build api client.
105   """
106   return build(serviceName='androidbuildinternal', version='v3', http=http,
107                static_discovery=False)
108 
109 
110 def create_client_from_json_keyfile(json_keyfile_name=None):
111   """Creates an Android build api client from a json keyfile.
112 
113   Args:
114     json_keyfile_name: The location of the keyfile, if None is provided use
115                        default location.
116 
117   Returns:
118     An authorized android build api client.
119   """
120   if not json_keyfile_name:
121     json_keyfile_name = os.path.join(os.getenv('HOME'), _DEF_JSON_KEYFILE)
122 
123   credentials = ServiceAccountCredentials.from_json_keyfile_name(
124       filename=json_keyfile_name, scopes=_SCOPE_URL)
125   http = httplib2.Http()
126   credentials.authorize(http)
127   return create_client(http)
128 
129 
130 def create_client_from_p12(robot_credentials_file, robot_username):
131   """Creates an Android build api client from a config file.
132 
133   Args:
134     robot_credentials_file: The path to the robot credentials file.
135     robot_username: A string containing the username of the robot account.
136 
137   Returns:
138     An authorized android build api client.
139   """
140   http = _create_http_from_p12(robot_credentials_file, robot_username)
141   return create_client(http)
142 
143 
144 def fetch_artifact(client, build_id, target, resource_id, dest):
145   """Fetches an artifact.
146 
147   Args:
148     client: An authorized android build api client.
149     build_id: AB build id
150     target: the target name to download from
151     resource_id: the resource id of the artifact
152     dest: path to store the artifact
153   """
154   out_dir = os.path.dirname(dest)
155   if not os.path.exists(out_dir):
156     os.makedirs(out_dir)
157 
158   dl_req = client.buildartifact().get_media(
159       buildId=build_id,
160       target=target,
161       attemptId='latest',
162       resourceId=resource_id)
163 
164   print('Fetching %s to %s...' % (resource_id, dest))
165   with io.FileIO(dest, mode='wb') as fh:
166     downloader = apiclient.http.MediaIoBaseDownload(
167         fh, dl_req, chunksize=_DEFAULT_CHUNK_SIZE)
168     done = False
169     while not done:
170       status, done = downloader.next_chunk(num_retries=_DEFAULT_RETRIES)
171       print('Fetching...' + str(status.progress() * 100))
172 
173   print('Done Fetching %s to %s' % (resource_id, dest))
174 
175 
176 def get_build_list(client, **kwargs):
177   """Get a list of builds from the android build api that matches parameters.
178 
179   Args:
180     client: An authorized android build api client.
181     **kwargs: keyworded arguments to pass to build api.
182 
183   Returns:
184     Response from build api.
185   """
186   build_request = client.build().list(**kwargs)
187 
188   return _simple_execute(build_request)
189 
190 
191 def list_artifacts(client, regex, **kwargs):
192   """List artifacts from the android build api that matches parameters.
193 
194   Args:
195     client: An authorized android build api client.
196     regex: Regular expression pattern to match artifact name.
197     **kwargs: keyworded arguments to pass to buildartifact.list api.
198 
199   Returns:
200     List of matching artifact names.
201   """
202   matching_artifacts = []
203   kwargs.setdefault('attemptId', 'latest')
204   req = client.buildartifact().list(nameRegexp=regex, **kwargs)
205   while req:
206     result = _simple_execute(req)
207     if result and 'artifacts' in result:
208       for a in result['artifacts']:
209         matching_artifacts.append(a['name'])
210     req = client.buildartifact().list_next(req, result)
211   return matching_artifacts
212 
213 
214 def fetch_artifacts(client, out_dir, target, pattern, build_id):
215   """Fetches target files artifacts matching patterns.
216 
217   Args:
218     client: An authorized instance of an android build api client for making
219       requests.
220     out_dir: The directory to store the fetched artifacts to.
221     target: The target name to download from.
222     pattern: A regex pattern to match to artifacts filename.
223     build_id: The Android Build id.
224   """
225   if not os.path.exists(out_dir):
226     os.makedirs(out_dir)
227 
228   # Build a list of needed artifacts
229   artifacts = list_artifacts(
230       client=client,
231       regex=pattern,
232       buildId=build_id,
233       target=target)
234 
235   for artifact in artifacts:
236     fetch_artifact(
237         client=client,
238         build_id=build_id,
239         target=target,
240         resource_id=artifact,
241         dest=os.path.join(out_dir, artifact))
242 
243 
244 def get_latest_build_id(client, branch, target):
245   """Get the latest build id.
246 
247   Args:
248     client: An authorized instance of an android build api client for making
249       requests.
250     branch: The branch to download from
251     target: The target name to download from.
252   Returns:
253     The build id.
254   """
255   build_response = get_build_list(
256       client=client,
257       branch=branch,
258       target=target,
259       maxResults=1,
260       successful=True,
261       buildType='submitted')
262 
263   if not build_response:
264     raise ValueError('Unable to determine latest build ID!')
265 
266   return build_response['builds'][0]['buildId']
267 
268 
269 def fetch_latest_artifacts(client, out_dir, target, pattern, branch):
270   """Fetches target files artifacts matching patterns from the latest build.
271 
272   Args:
273     client: An authorized instance of an android build api client for making
274       requests.
275     out_dir: The directory to store the fetched artifacts to.
276     target: The target name to download from.
277     pattern: A regex pattern to match to artifacts filename
278     branch: The branch to download from
279   """
280   build_id = get_latest_build_id(
281       client=client, branch=branch, target=target)
282 
283   fetch_artifacts(client, out_dir, target, pattern, build_id)
284