Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 1 | # Copyright 2015 Google Inc. All rights reserved. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | |
| 15 | """Installs an Android application, possibly in an incremental way.""" |
| 16 | |
| 17 | import collections |
| 18 | import hashlib |
| 19 | import logging |
| 20 | import os |
| 21 | import re |
| 22 | import shutil |
| 23 | import subprocess |
| 24 | import sys |
| 25 | import tempfile |
| 26 | import time |
| 27 | import zipfile |
| 28 | |
| 29 | from third_party.py import gflags |
| 30 | from third_party.py.concurrent import futures |
| 31 | |
| 32 | |
| 33 | gflags.DEFINE_string("split_main_apk", None, "The main APK for split install") |
| 34 | gflags.DEFINE_multistring("split_apk", [], "Split APKs to install") |
| 35 | gflags.DEFINE_string("dexmanifest", None, "The .dex manifest") |
Lukacs Berki | ecbf242 | 2015-07-28 07:57:45 +0000 | [diff] [blame] | 36 | gflags.DEFINE_multistring("native_lib", None, "Native libraries to install") |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 37 | gflags.DEFINE_string("resource_apk", None, "The resource .apk") |
| 38 | gflags.DEFINE_string("apk", None, "The app .apk. If not specified, " |
| 39 | "do incremental deployment") |
| 40 | gflags.DEFINE_string("adb", None, "ADB to use") |
| 41 | gflags.DEFINE_string("stub_datafile", None, "The stub data file") |
| 42 | gflags.DEFINE_string("output_marker", None, "The output marker file") |
| 43 | gflags.DEFINE_multistring("extra_adb_arg", [], "Extra arguments to adb") |
| 44 | gflags.DEFINE_string("execroot", ".", "The exec root") |
| 45 | gflags.DEFINE_integer("adb_jobs", 2, |
| 46 | "The number of instances of adb to use in parallel to " |
| 47 | "update files on the device", |
| 48 | lower_bound=1) |
Googler | 73d4fc9 | 2015-07-30 20:39:46 +0000 | [diff] [blame] | 49 | gflags.DEFINE_enum("start", "no", ["no", "cold", "warm"], "Whether/how to " |
| 50 | "start the app after installing it. 'cold' and 'warm' will " |
| 51 | "both cause the app to be started, 'warm' will start it " |
| 52 | "with previously saved application state.") |
| 53 | gflags.DEFINE_boolean("start_app", False, "Deprecated, use 'start'.") |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 54 | gflags.DEFINE_string("user_home_dir", None, "Path to the user's home directory") |
| 55 | gflags.DEFINE_string("flagfile", None, |
| 56 | "Path to a file to read additional flags from") |
| 57 | gflags.DEFINE_string("verbosity", None, "Logging verbosity") |
| 58 | |
| 59 | FLAGS = gflags.FLAGS |
| 60 | |
| 61 | DEVICE_DIRECTORY = "/data/local/tmp/incrementaldeployment" |
| 62 | |
Lukacs Berki | a3a33d7 | 2015-08-19 08:34:55 +0000 | [diff] [blame] | 63 | # Some devices support ABIs other than those reported by getprop. In this case, |
| 64 | # if the most specific ABI is not available in the .apk, we push the more |
| 65 | # general ones. |
| 66 | COMPATIBLE_ABIS = { |
| 67 | "armeabi-v7a": ["armeabi"], |
| 68 | "arm64-v8a": ["armeabi-v7a", "armeabi"] |
| 69 | } |
| 70 | |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 71 | |
| 72 | class AdbError(Exception): |
| 73 | """An exception class signaling an error in an adb invocation.""" |
| 74 | |
| 75 | def __init__(self, args, returncode, stdout, stderr): |
| 76 | self.args = args |
| 77 | self.returncode = returncode |
| 78 | self.stdout = stdout |
| 79 | self.stderr = stderr |
| 80 | details = "\n".join([ |
| 81 | "adb command: %s" % args, |
| 82 | "return code: %s" % returncode, |
| 83 | "stdout: %s" % stdout, |
| 84 | "stderr: %s" % stderr, |
| 85 | ]) |
| 86 | super(AdbError, self).__init__(details) |
| 87 | |
| 88 | |
| 89 | class DeviceNotFoundError(Exception): |
| 90 | """Raised when the device could not be found.""" |
| 91 | |
| 92 | |
| 93 | class MultipleDevicesError(Exception): |
| 94 | """Raised when > 1 device is attached and no device serial was given.""" |
| 95 | |
| 96 | @staticmethod |
| 97 | def CheckError(s): |
| 98 | return re.search("more than one (device and emulator|device|emulator)", s) |
| 99 | |
| 100 | |
| 101 | class DeviceUnauthorizedError(Exception): |
| 102 | """Raised when the local machine is not authorized to the device.""" |
| 103 | |
| 104 | |
| 105 | class TimestampException(Exception): |
| 106 | """Raised when there is a problem with timestamp reading/writing.""" |
| 107 | |
| 108 | |
| 109 | class Adb(object): |
| 110 | """A class to handle interaction with adb.""" |
| 111 | |
| 112 | def __init__(self, adb_path, temp_dir, adb_jobs, user_home_dir): |
| 113 | self._adb_path = adb_path |
| 114 | self._temp_dir = temp_dir |
| 115 | self._user_home_dir = user_home_dir |
| 116 | self._file_counter = 1 |
| 117 | self._executor = futures.ThreadPoolExecutor(max_workers=adb_jobs) |
| 118 | |
| 119 | def _Exec(self, adb_args): |
| 120 | """Executes the given adb command + args.""" |
| 121 | args = [self._adb_path] + FLAGS.extra_adb_arg + adb_args |
| 122 | # TODO(ahumesky): Because multiple instances of adb are executed in |
| 123 | # parallel, these debug logging lines will get interleaved. |
| 124 | logging.debug("Executing: %s", " ".join(args)) |
| 125 | |
| 126 | # adb sometimes requires the user's home directory to access things in |
| 127 | # $HOME/.android (e.g. keys to authorize with the device). To avoid any |
| 128 | # potential problems with python picking up things in the user's home |
| 129 | # directory, HOME is not set in the environment around python and is instead |
| 130 | # passed explicitly as a flag. |
| 131 | env = {} |
| 132 | if self._user_home_dir: |
| 133 | env["HOME"] = self._user_home_dir |
| 134 | |
| 135 | adb = subprocess.Popen( |
| 136 | args, |
| 137 | stdin=subprocess.PIPE, |
| 138 | stdout=subprocess.PIPE, |
| 139 | stderr=subprocess.PIPE, |
| 140 | env=env) |
| 141 | stdout, stderr = adb.communicate() |
| 142 | stdout = stdout.strip() |
| 143 | stderr = stderr.strip() |
| 144 | logging.debug("adb ret: %s", adb.returncode) |
| 145 | logging.debug("adb out: %s", stdout) |
| 146 | logging.debug("adb err: %s", stderr) |
| 147 | |
| 148 | # Check these first so that the more specific error gets raised instead of |
| 149 | # the more generic AdbError. |
| 150 | if "device not found" in stderr: |
| 151 | raise DeviceNotFoundError() |
| 152 | elif "device unauthorized" in stderr: |
| 153 | raise DeviceUnauthorizedError() |
| 154 | elif MultipleDevicesError.CheckError(stderr): |
| 155 | # The error messages are from adb's transport.c, but something adds |
| 156 | # "error: " to the beginning, so take it off so that we don't end up |
| 157 | # printing "Error: error: ..." |
| 158 | raise MultipleDevicesError(re.sub("^error: ", "", stderr)) |
| 159 | |
| 160 | if adb.returncode != 0: |
| 161 | raise AdbError(args, adb.returncode, stdout, stderr) |
| 162 | |
| 163 | return adb.returncode, stdout, stderr, args |
| 164 | |
| 165 | def _ExecParallel(self, adb_args): |
| 166 | return self._executor.submit(self._Exec, adb_args) |
| 167 | |
| 168 | def _CreateLocalFile(self): |
| 169 | """Returns a path to a temporary local file in the temp directory.""" |
| 170 | local = os.path.join(self._temp_dir, "adbfile_%d" % self._file_counter) |
| 171 | self._file_counter += 1 |
| 172 | return local |
| 173 | |
| 174 | def GetInstallTime(self, package): |
| 175 | """Get the installation time of a package.""" |
| 176 | _, stdout, _, _ = self._Shell("dumpsys package %s" % package) |
Lukacs Berki | faff718 | 2015-08-20 14:27:42 +0000 | [diff] [blame] | 177 | match = re.search("firstInstallTime=(.*)$", stdout, re.MULTILINE) |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 178 | if match: |
| 179 | return match.group(1) |
| 180 | else: |
Lukacs Berki | 0dffeac | 2015-09-15 07:18:47 +0000 | [diff] [blame] | 181 | return None |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 182 | |
Lukacs Berki | ecbf242 | 2015-07-28 07:57:45 +0000 | [diff] [blame] | 183 | def GetAbi(self): |
| 184 | """Returns the ABI the device supports.""" |
| 185 | _, stdout, _, _ = self._Shell("getprop ro.product.cpu.abi") |
| 186 | return stdout |
| 187 | |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 188 | def Push(self, local, remote): |
| 189 | """Invoke 'adb push' in parallel.""" |
| 190 | return self._ExecParallel(["push", local, remote]) |
| 191 | |
| 192 | def PushString(self, contents, remote): |
| 193 | """Push a given string to a given path on the device in parallel.""" |
| 194 | local = self._CreateLocalFile() |
| 195 | with file(local, "w") as f: |
| 196 | f.write(contents) |
| 197 | return self.Push(local, remote) |
| 198 | |
| 199 | def Pull(self, remote): |
| 200 | """Invoke 'adb pull'. |
| 201 | |
| 202 | Args: |
| 203 | remote: The path to the remote file to pull. |
| 204 | |
| 205 | Returns: |
| 206 | The contents of a file or None if the file didn't exist. |
| 207 | """ |
| 208 | local = self._CreateLocalFile() |
| 209 | try: |
| 210 | self._Exec(["pull", remote, local]) |
| 211 | with file(local) as f: |
| 212 | return f.read() |
| 213 | except (AdbError, IOError): |
| 214 | return None |
| 215 | |
| 216 | def InstallMultiple(self, apk, pkg=None): |
| 217 | """Invoke 'adb install-multiple'.""" |
| 218 | |
| 219 | pkg_args = ["-p", pkg] if pkg else [] |
| 220 | ret, stdout, stderr, args = self._Exec( |
| 221 | ["install-multiple", "-r"] + pkg_args + [apk]) |
| 222 | if "Success" not in stderr and "Success" not in stdout: |
| 223 | raise AdbError(args, ret, stdout, stderr) |
| 224 | |
| 225 | def Install(self, apk): |
| 226 | """Invoke 'adb install'.""" |
| 227 | ret, stdout, stderr, args = self._Exec(["install", "-r", apk]) |
| 228 | |
| 229 | # adb install could fail with a message on stdout like this: |
| 230 | # |
| 231 | # pkg: /data/local/tmp/Gmail_dev_sharded_incremental.apk |
| 232 | # Failure [INSTALL_PARSE_FAILED_INCONSISTENT_CERTIFICATES] |
| 233 | # |
| 234 | # and yet it will still have a return code of 0. At least for the install |
| 235 | # command, it will print "Success" if it succeeded, so check for that in |
| 236 | # standard out instead of relying on the return code. |
| 237 | if "Success" not in stderr and "Success" not in stdout: |
| 238 | raise AdbError(args, ret, stdout, stderr) |
| 239 | |
Lukacs Berki | faff718 | 2015-08-20 14:27:42 +0000 | [diff] [blame] | 240 | def Uninstall(self, pkg): |
| 241 | """Invoke 'adb uninstall'.""" |
| 242 | self._Exec(["uninstall", pkg]) |
| 243 | # No error checking. If this fails, we assume that the app was not installed |
| 244 | # in the first place. |
| 245 | |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 246 | def Delete(self, remote): |
| 247 | """Delete the given file (or directory) on the device.""" |
| 248 | self.DeleteMultiple([remote]) |
| 249 | |
| 250 | def DeleteMultiple(self, remote_files): |
| 251 | """Delete the given files (or directories) on the device.""" |
| 252 | files_str = " ".join(remote_files) |
| 253 | if files_str: |
| 254 | self._Shell("rm -fr %s" % files_str) |
| 255 | |
| 256 | def Mkdir(self, d): |
| 257 | """Invokes mkdir with the specified directory on the device.""" |
| 258 | self._Shell("mkdir -p %s" % d) |
| 259 | |
| 260 | def StopApp(self, package): |
| 261 | """Force stops the app with the given package.""" |
| 262 | self._Shell("am force-stop %s" % package) |
| 263 | |
Googler | 73d4fc9 | 2015-07-30 20:39:46 +0000 | [diff] [blame] | 264 | def StopAppAndSaveState(self, package): |
| 265 | """Stops the app with the given package, saving state for the next run.""" |
| 266 | # 'am kill' will only kill processes in the background, so we must make sure |
| 267 | # our process is in the background first. We accomplish this by bringing up |
| 268 | # the app switcher. |
| 269 | self._Shell("input keyevent KEYCODE_APP_SWITCH") |
| 270 | self._Shell("am kill %s" % package) |
| 271 | |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 272 | def StartApp(self, package): |
| 273 | """Starts the app with the given package.""" |
| 274 | self._Shell("monkey -p %s -c android.intent.category.LAUNCHER 1" % package) |
| 275 | |
| 276 | def _Shell(self, cmd): |
| 277 | """Invoke 'adb shell'.""" |
| 278 | return self._Exec(["shell", cmd]) |
| 279 | |
| 280 | |
| 281 | ManifestEntry = collections.namedtuple( |
| 282 | "ManifestEntry", ["input_file", "zippath", "installpath", "sha256"]) |
| 283 | |
| 284 | |
| 285 | def ParseManifest(contents): |
| 286 | """Parses a dexmanifest file. |
| 287 | |
| 288 | Args: |
| 289 | contents: the contents of the manifest file to be parsed. |
| 290 | |
| 291 | Returns: |
| 292 | A dict of install path -> ManifestEntry. |
| 293 | """ |
| 294 | result = {} |
| 295 | |
| 296 | for l in contents.split("\n"): |
| 297 | entry = ManifestEntry(*(l.strip().split(" "))) |
| 298 | result[entry.installpath] = entry |
| 299 | |
| 300 | return result |
| 301 | |
| 302 | |
| 303 | def GetAppPackage(stub_datafile): |
| 304 | """Returns the app package specified in a stub data file.""" |
| 305 | with file(stub_datafile) as f: |
| 306 | return f.readlines()[1].strip() |
| 307 | |
| 308 | |
| 309 | def UploadDexes(adb, execroot, app_dir, temp_dir, dexmanifest, full_install): |
| 310 | """Uploads dexes to the device so that the state. |
| 311 | |
| 312 | Does the minimum amount of work necessary to make the state of the device |
| 313 | consistent with what was built. |
| 314 | |
| 315 | Args: |
| 316 | adb: the Adb instance representing the device to install to |
| 317 | execroot: the execroot |
| 318 | app_dir: the directory things should be installed under on the device |
| 319 | temp_dir: a local temporary directory |
| 320 | dexmanifest: contents of the dex manifest |
| 321 | full_install: whether to do a full install |
| 322 | |
| 323 | Returns: |
| 324 | None. |
| 325 | """ |
| 326 | |
| 327 | # Fetch the manifest on the device |
| 328 | dex_dir = os.path.join(app_dir, "dex") |
| 329 | adb.Mkdir(dex_dir) |
| 330 | |
| 331 | old_manifest = None |
| 332 | |
| 333 | if not full_install: |
| 334 | logging.info("Fetching dex manifest from device...") |
| 335 | old_manifest_contents = adb.Pull("%s/manifest" % dex_dir) |
| 336 | if old_manifest_contents: |
| 337 | old_manifest = ParseManifest(old_manifest_contents) |
| 338 | else: |
| 339 | logging.info("Dex manifest not found on device") |
| 340 | |
| 341 | if old_manifest is None: |
| 342 | # If the manifest is not found, maybe a previous installation attempt |
| 343 | # was interrupted. Wipe the slate clean. Do this also in case we do a full |
| 344 | # installation. |
| 345 | old_manifest = {} |
| 346 | adb.Delete("%s/*" % dex_dir) |
| 347 | |
| 348 | new_manifest = ParseManifest(dexmanifest) |
| 349 | dexes_to_delete = set(old_manifest) - set(new_manifest) |
| 350 | |
| 351 | # Figure out which dexes to upload: those that are present in the new manifest |
| 352 | # but not in the old one and those whose checksum was changed |
| 353 | common_dexes = set(new_manifest).intersection(old_manifest) |
| 354 | dexes_to_upload = set(d for d in common_dexes |
| 355 | if new_manifest[d].sha256 != old_manifest[d].sha256) |
| 356 | dexes_to_upload.update(set(new_manifest) - set(old_manifest)) |
| 357 | |
| 358 | if not dexes_to_delete and not dexes_to_upload: |
| 359 | # If we have nothing to do, don't bother removing and rewriting the manifest |
| 360 | logging.info("Application dexes up-to-date") |
| 361 | return |
| 362 | |
| 363 | # Delete the manifest so that we know how to get back to a consistent state |
| 364 | # if we are interrupted. |
| 365 | adb.Delete("%s/manifest" % dex_dir) |
| 366 | |
| 367 | # Tuple of (local, remote) files to push to the device. |
| 368 | files_to_push = [] |
| 369 | |
| 370 | # Sort dexes to be uploaded by the zip file they are in so that we only need |
| 371 | # to open each zip only once. |
| 372 | dexzips_in_upload = set(new_manifest[d].input_file for d in dexes_to_upload |
| 373 | if new_manifest[d].zippath != "-") |
| 374 | for i, dexzip_name in enumerate(dexzips_in_upload): |
| 375 | zip_dexes = [ |
| 376 | d for d in dexes_to_upload if new_manifest[d].input_file == dexzip_name] |
| 377 | dexzip_tempdir = os.path.join(temp_dir, "dex", str(i)) |
| 378 | with zipfile.ZipFile(os.path.join(execroot, dexzip_name)) as dexzip: |
| 379 | for dex in zip_dexes: |
| 380 | zippath = new_manifest[dex].zippath |
| 381 | dexzip.extract(zippath, dexzip_tempdir) |
| 382 | files_to_push.append( |
| 383 | (os.path.join(dexzip_tempdir, zippath), "%s/%s" % (dex_dir, dex))) |
| 384 | |
| 385 | # Now gather all the dexes that are not within a .zip file. |
| 386 | dexes_to_upload = set( |
| 387 | d for d in dexes_to_upload if new_manifest[d].zippath == "-") |
| 388 | for dex in dexes_to_upload: |
| 389 | files_to_push.append( |
| 390 | (new_manifest[dex].input_file, "%s/%s" % (dex_dir, dex))) |
| 391 | |
| 392 | num_files = len(dexes_to_delete) + len(files_to_push) |
| 393 | logging.info("Updating %d dex%s...", num_files, "es" if num_files > 1 else "") |
| 394 | |
| 395 | # Delete the dexes that are not in the new manifest |
| 396 | adb.DeleteMultiple(os.path.join(dex_dir, dex) for dex in dexes_to_delete) |
| 397 | |
| 398 | # Upload all the files. |
| 399 | upload_walltime_start = time.time() |
| 400 | fs = [adb.Push(local, remote) for local, remote in files_to_push] |
| 401 | done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION) |
| 402 | upload_walltime = time.time() - upload_walltime_start |
| 403 | logging.debug("Dex upload walltime: %s seconds", upload_walltime) |
| 404 | |
| 405 | # If there is anything in not_done, then some adb call failed and we |
| 406 | # can cancel the rest. |
| 407 | if not_done: |
| 408 | for f in not_done: |
| 409 | f.cancel() |
| 410 | |
| 411 | # If any adb call resulted in an exception, re-raise it. |
| 412 | for f in done: |
| 413 | f.result() |
| 414 | |
| 415 | # If no dex upload failed, upload the manifest. If any upload failed, the |
| 416 | # exception should have been re-raised above. |
| 417 | # Call result() to raise the exception if there was one. |
| 418 | adb.PushString(dexmanifest, "%s/manifest" % dex_dir).result() |
| 419 | |
| 420 | |
| 421 | def Checksum(filename): |
| 422 | """Compute the SHA-256 checksum of a file.""" |
| 423 | h = hashlib.sha256() |
| 424 | with file(filename, "r") as f: |
| 425 | while True: |
| 426 | data = f.read(65536) |
| 427 | if not data: |
| 428 | break |
| 429 | |
| 430 | h.update(data) |
| 431 | |
| 432 | return h.hexdigest() |
| 433 | |
| 434 | |
| 435 | def UploadResources(adb, resource_apk, app_dir): |
| 436 | """Uploads resources to the device. |
| 437 | |
| 438 | Args: |
| 439 | adb: The Adb instance representing the device to install to. |
| 440 | resource_apk: Path to the resource apk. |
| 441 | app_dir: The directory things should be installed under on the device. |
| 442 | |
| 443 | Returns: |
| 444 | None. |
| 445 | """ |
| 446 | |
| 447 | # Compute the checksum of the new resources file |
| 448 | new_checksum = Checksum(resource_apk) |
| 449 | |
| 450 | # Fetch the checksum of the resources file on the device, if it exists |
| 451 | device_checksum_file = "%s/%s" % (app_dir, "resources_checksum") |
| 452 | old_checksum = adb.Pull(device_checksum_file) |
| 453 | if old_checksum == new_checksum: |
| 454 | logging.info("Application resources up-to-date") |
| 455 | return |
| 456 | logging.info("Updating application resources...") |
| 457 | |
| 458 | # Remove the checksum file on the device so that if the transfer is |
| 459 | # interrupted, we know how to get the device back to a consistent state. |
| 460 | adb.Delete(device_checksum_file) |
| 461 | adb.Push(resource_apk, "%s/%s" % (app_dir, "resources.ap_")).result() |
| 462 | |
| 463 | # Write the new checksum to the device. |
| 464 | adb.PushString(new_checksum, device_checksum_file).result() |
| 465 | |
| 466 | |
Lukacs Berki | ecbf242 | 2015-07-28 07:57:45 +0000 | [diff] [blame] | 467 | def ConvertNativeLibs(args): |
| 468 | """Converts the --native_libs command line argument to an arch -> libs map.""" |
| 469 | native_libs = {} |
| 470 | if args is not None: |
| 471 | for native_lib in args: |
| 472 | abi, path = native_lib.split(":") |
| 473 | if abi not in native_libs: |
| 474 | native_libs[abi] = set() |
| 475 | |
| 476 | native_libs[abi].add(path) |
| 477 | |
| 478 | return native_libs |
| 479 | |
| 480 | |
Lukacs Berki | a3a33d7 | 2015-08-19 08:34:55 +0000 | [diff] [blame] | 481 | def FindAbi(device_abi, app_abis): |
| 482 | """Selects which ABI native libs should be installed for.""" |
| 483 | if device_abi in app_abis: |
| 484 | return device_abi |
| 485 | |
| 486 | if device_abi in COMPATIBLE_ABIS: |
| 487 | for abi in COMPATIBLE_ABIS[device_abi]: |
| 488 | if abi in app_abis: |
| 489 | logging.warn("App does not have native libs for ABI '%s'. Using ABI " |
| 490 | "'%s'.", device_abi, abi) |
| 491 | return abi |
| 492 | |
| 493 | logging.warn("No native libs for device ABI '%s'. App has native libs for " |
| 494 | "ABIs: %s", device_abi, ", ".join(app_abis)) |
| 495 | return None |
| 496 | |
| 497 | |
Lukacs Berki | b4b19bc | 2015-07-30 11:45:35 +0000 | [diff] [blame] | 498 | def UploadNativeLibs(adb, native_lib_args, app_dir, full_install): |
Lukacs Berki | ecbf242 | 2015-07-28 07:57:45 +0000 | [diff] [blame] | 499 | """Uploads native libraries to the device.""" |
| 500 | |
| 501 | native_libs = ConvertNativeLibs(native_lib_args) |
| 502 | libs = set() |
| 503 | if native_libs: |
Lukacs Berki | a3a33d7 | 2015-08-19 08:34:55 +0000 | [diff] [blame] | 504 | abi = FindAbi(adb.GetAbi(), native_libs.keys()) |
| 505 | if abi: |
Lukacs Berki | ecbf242 | 2015-07-28 07:57:45 +0000 | [diff] [blame] | 506 | libs = native_libs[abi] |
| 507 | |
| 508 | basename_to_path = {} |
| 509 | install_checksums = {} |
| 510 | for lib in sorted(libs): |
| 511 | install_checksums[os.path.basename(lib)] = Checksum(lib) |
| 512 | basename_to_path[os.path.basename(lib)] = lib |
| 513 | |
Lukacs Berki | b4b19bc | 2015-07-30 11:45:35 +0000 | [diff] [blame] | 514 | device_manifest = None |
| 515 | if not full_install: |
| 516 | device_manifest = adb.Pull("%s/native/native_manifest" % app_dir) |
| 517 | |
Lukacs Berki | ecbf242 | 2015-07-28 07:57:45 +0000 | [diff] [blame] | 518 | device_checksums = {} |
Lukacs Berki | b4b19bc | 2015-07-30 11:45:35 +0000 | [diff] [blame] | 519 | if device_manifest is None: |
| 520 | # If we couldn't fetch the device manifest or if this is a non-incremental |
| 521 | # install, wipe the slate clean |
| 522 | adb.Delete("%s/native" % app_dir) |
| 523 | else: |
| 524 | # Otherwise, parse the manifest. Note that this branch is also taken if the |
| 525 | # manifest is empty. |
Lukacs Berki | ff6ef9b | 2015-08-14 08:17:41 +0000 | [diff] [blame] | 526 | for manifest_line in device_manifest.split("\n"): |
| 527 | if manifest_line: |
| 528 | name, checksum = manifest_line.split(" ") |
| 529 | device_checksums[name] = checksum |
Lukacs Berki | ecbf242 | 2015-07-28 07:57:45 +0000 | [diff] [blame] | 530 | |
| 531 | libs_to_delete = set(device_checksums) - set(install_checksums) |
| 532 | libs_to_upload = set(install_checksums) - set(device_checksums) |
| 533 | common_libs = set(install_checksums).intersection(set(device_checksums)) |
| 534 | libs_to_upload.update([l for l in common_libs |
| 535 | if install_checksums[l] != device_checksums[l]]) |
| 536 | |
| 537 | libs_to_push = [(basename_to_path[lib], "%s/native/%s" % (app_dir, lib)) |
| 538 | for lib in libs_to_upload] |
| 539 | |
| 540 | if not libs_to_delete and not libs_to_push and device_manifest is not None: |
| 541 | logging.info("Native libs up-to-date") |
| 542 | return |
| 543 | |
| 544 | num_files = len(libs_to_delete) + len(libs_to_push) |
| 545 | logging.info("Updating %d native lib%s...", |
| 546 | num_files, "s" if num_files != 1 else "") |
| 547 | |
| 548 | adb.Delete("%s/native/native_manifest" % app_dir) |
| 549 | |
| 550 | if libs_to_delete: |
| 551 | adb.DeleteMultiple([ |
| 552 | "%s/native/%s" % (app_dir, lib) for lib in libs_to_delete]) |
| 553 | |
| 554 | upload_walltime_start = time.time() |
| 555 | fs = [adb.Push(local, remote) for local, remote in libs_to_push] |
| 556 | done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION) |
| 557 | upload_walltime = time.time() - upload_walltime_start |
| 558 | logging.debug("Native library upload walltime: %s seconds", upload_walltime) |
| 559 | |
| 560 | # If there is anything in not_done, then some adb call failed and we |
| 561 | # can cancel the rest. |
| 562 | if not_done: |
| 563 | for f in not_done: |
| 564 | f.cancel() |
| 565 | |
| 566 | # If any adb call resulted in an exception, re-raise it. |
| 567 | for f in done: |
| 568 | f.result() |
| 569 | |
| 570 | install_manifest = [ |
| 571 | name + " " + checksum for name, checksum in install_checksums.iteritems()] |
| 572 | adb.PushString("\n".join(install_manifest), |
| 573 | "%s/native/native_manifest" % app_dir).result() |
| 574 | |
| 575 | |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 576 | def VerifyInstallTimestamp(adb, app_package): |
| 577 | """Verifies that the app is unchanged since the last mobile-install.""" |
| 578 | expected_timestamp = adb.Pull("%s/%s/install_timestamp" % ( |
| 579 | DEVICE_DIRECTORY, app_package)) |
| 580 | if not expected_timestamp: |
| 581 | raise TimestampException( |
| 582 | "Cannot verify last mobile install. At least one non-incremental " |
| 583 | "'mobile-install' must precede incremental installs") |
| 584 | |
| 585 | actual_timestamp = adb.GetInstallTime(app_package) |
Lukacs Berki | 0dffeac | 2015-09-15 07:18:47 +0000 | [diff] [blame] | 586 | if actual_timestamp is None: |
| 587 | raise TimestampException( |
| 588 | "Package '%s' is not installed on the device. At least one " |
| 589 | "non-incremental 'mobile-install' must precede incremental " |
| 590 | "installs." % app_package) |
| 591 | |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 592 | if actual_timestamp != expected_timestamp: |
| 593 | raise TimestampException("Installed app '%s' has an unexpected timestamp. " |
| 594 | "Did you last install the app in a way other than " |
| 595 | "'mobile-install'?" % app_package) |
| 596 | |
| 597 | |
Lukacs Berki | faff718 | 2015-08-20 14:27:42 +0000 | [diff] [blame] | 598 | def SplitIncrementalInstall(adb, app_package, execroot, split_main_apk, |
| 599 | split_apks): |
| 600 | """Does incremental installation using split packages.""" |
| 601 | app_dir = os.path.join(DEVICE_DIRECTORY, app_package) |
| 602 | device_manifest_path = "%s/split_manifest" % app_dir |
| 603 | device_manifest = adb.Pull(device_manifest_path) |
| 604 | expected_timestamp = adb.Pull("%s/install_timestamp" % app_dir) |
| 605 | actual_timestamp = adb.GetInstallTime(app_package) |
| 606 | device_checksums = {} |
| 607 | if device_manifest is not None: |
| 608 | for manifest_line in device_manifest.split("\n"): |
| 609 | if manifest_line: |
| 610 | name, checksum = manifest_line.split(" ") |
| 611 | device_checksums[name] = checksum |
| 612 | |
| 613 | install_checksums = {} |
| 614 | install_checksums["__MAIN__"] = Checksum( |
| 615 | os.path.join(execroot, split_main_apk)) |
| 616 | for apk in split_apks: |
| 617 | install_checksums[apk] = Checksum(os.path.join(execroot, apk)) |
| 618 | |
| 619 | reinstall_main = False |
| 620 | if (device_manifest is None or actual_timestamp is None or |
| 621 | actual_timestamp != expected_timestamp or |
| 622 | install_checksums["__MAIN__"] != device_checksums["__MAIN__"] or |
| 623 | set(device_checksums.keys()) != set(install_checksums.keys())): |
| 624 | # The main app is not up to date or not present or something happened |
| 625 | # with the on-device manifest. Start from scratch. Notably, we cannot |
| 626 | # uninstall a split package, so if the set of packages changes, we also |
| 627 | # need to do a full reinstall. |
| 628 | reinstall_main = True |
| 629 | device_checksums = {} |
| 630 | |
| 631 | apks_to_update = [ |
| 632 | apk for apk in split_apks if |
| 633 | apk not in device_checksums or |
| 634 | device_checksums[apk] != install_checksums[apk]] |
| 635 | |
| 636 | if not apks_to_update and not reinstall_main: |
| 637 | # Nothing to do |
| 638 | return |
| 639 | |
| 640 | # Delete the device manifest so that if something goes wrong, we do a full |
| 641 | # reinstall next time |
| 642 | adb.Delete(device_manifest_path) |
| 643 | |
| 644 | if reinstall_main: |
| 645 | logging.info("Installing main APK...") |
| 646 | adb.Uninstall(app_package) |
| 647 | adb.InstallMultiple(os.path.join(execroot, split_main_apk)) |
| 648 | adb.PushString( |
| 649 | adb.GetInstallTime(app_package), |
| 650 | "%s/install_timestamp" % app_dir).result() |
| 651 | |
| 652 | logging.info("Reinstalling %s APKs...", len(apks_to_update)) |
| 653 | |
| 654 | for apk in apks_to_update: |
| 655 | adb.InstallMultiple(os.path.join(execroot, apk), app_package) |
| 656 | |
| 657 | install_manifest = [ |
| 658 | name + " " + checksum for name, checksum in install_checksums.iteritems()] |
| 659 | adb.PushString("\n".join(install_manifest), |
| 660 | "%s/split_manifest" % app_dir).result() |
| 661 | |
| 662 | |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 663 | def IncrementalInstall(adb_path, execroot, stub_datafile, output_marker, |
Googler | 73d4fc9 | 2015-07-30 20:39:46 +0000 | [diff] [blame] | 664 | adb_jobs, start_type, dexmanifest=None, apk=None, |
Lukacs Berki | ecbf242 | 2015-07-28 07:57:45 +0000 | [diff] [blame] | 665 | native_libs=None, resource_apk=None, |
| 666 | split_main_apk=None, split_apks=None, |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 667 | user_home_dir=None): |
| 668 | """Performs an incremental install. |
| 669 | |
| 670 | Args: |
| 671 | adb_path: Path to the adb executable. |
| 672 | execroot: Exec root. |
| 673 | stub_datafile: The stub datafile containing the app's package name. |
| 674 | output_marker: Path to the output marker file. |
| 675 | adb_jobs: The number of instances of adb to use in parallel. |
Googler | 73d4fc9 | 2015-07-30 20:39:46 +0000 | [diff] [blame] | 676 | start_type: A string describing whether/how to start the app after |
| 677 | installing it. Can be 'no', 'cold', or 'warm'. |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 678 | dexmanifest: Path to the .dex manifest file. |
| 679 | apk: Path to the .apk file. May be None to perform an incremental install. |
Lukacs Berki | ecbf242 | 2015-07-28 07:57:45 +0000 | [diff] [blame] | 680 | native_libs: Native libraries to install. |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 681 | resource_apk: Path to the apk containing the app's resources. |
| 682 | split_main_apk: the split main .apk if split installation is desired. |
| 683 | split_apks: the list of split .apks to be installed. |
| 684 | user_home_dir: Path to the user's home directory. |
| 685 | """ |
| 686 | temp_dir = tempfile.mkdtemp() |
| 687 | try: |
| 688 | adb = Adb(adb_path, temp_dir, adb_jobs, user_home_dir) |
| 689 | app_package = GetAppPackage(os.path.join(execroot, stub_datafile)) |
| 690 | app_dir = os.path.join(DEVICE_DIRECTORY, app_package) |
| 691 | if split_main_apk: |
Lukacs Berki | faff718 | 2015-08-20 14:27:42 +0000 | [diff] [blame] | 692 | SplitIncrementalInstall(adb, app_package, execroot, split_main_apk, |
| 693 | split_apks) |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 694 | else: |
| 695 | if not apk: |
| 696 | VerifyInstallTimestamp(adb, app_package) |
| 697 | |
| 698 | with file(os.path.join(execroot, dexmanifest)) as f: |
| 699 | dexmanifest = f.read() |
| 700 | UploadDexes(adb, execroot, app_dir, temp_dir, dexmanifest, bool(apk)) |
| 701 | # TODO(ahumesky): UploadDexes waits for all the dexes to be uploaded, and |
| 702 | # then UploadResources is called. We could instead enqueue everything |
| 703 | # onto the threadpool so that uploading resources happens sooner. |
| 704 | UploadResources(adb, os.path.join(execroot, resource_apk), app_dir) |
Lukacs Berki | b4b19bc | 2015-07-30 11:45:35 +0000 | [diff] [blame] | 705 | UploadNativeLibs(adb, native_libs, app_dir, bool(apk)) |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 706 | if apk: |
| 707 | apk_path = os.path.join(execroot, apk) |
| 708 | adb.Install(apk_path) |
| 709 | future = adb.PushString( |
| 710 | adb.GetInstallTime(app_package), |
| 711 | "%s/%s/install_timestamp" % (DEVICE_DIRECTORY, app_package)) |
| 712 | future.result() |
| 713 | |
| 714 | else: |
Googler | 73d4fc9 | 2015-07-30 20:39:46 +0000 | [diff] [blame] | 715 | if start_type == "warm": |
| 716 | adb.StopAppAndSaveState(app_package) |
| 717 | else: |
| 718 | adb.StopApp(app_package) |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 719 | |
Googler | 73d4fc9 | 2015-07-30 20:39:46 +0000 | [diff] [blame] | 720 | if start_type in ["cold", "warm"]: |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 721 | logging.info("Starting application %s", app_package) |
| 722 | adb.StartApp(app_package) |
| 723 | |
| 724 | with file(output_marker, "w") as _: |
| 725 | pass |
| 726 | except DeviceNotFoundError: |
| 727 | sys.exit("Error: Device not found") |
| 728 | except DeviceUnauthorizedError: |
| 729 | sys.exit("Error: Device unauthorized. Please check the confirmation " |
| 730 | "dialog on your device.") |
| 731 | except MultipleDevicesError as e: |
| 732 | sys.exit( |
| 733 | "Error: " + e.message + "\nTry specifying a device serial with " + |
| 734 | "\"blaze mobile-install --adb_arg=-s --adb_arg=$ANDROID_SERIAL\"") |
| 735 | except TimestampException as e: |
| 736 | sys.exit("Error:\n%s" % e.message) |
| 737 | except AdbError as e: |
| 738 | sys.exit("Error:\n%s" % e.message) |
| 739 | finally: |
| 740 | shutil.rmtree(temp_dir, True) |
| 741 | |
| 742 | |
| 743 | def main(): |
| 744 | if FLAGS.verbosity == "1": |
| 745 | level = logging.DEBUG |
| 746 | fmt = "%(levelname)-5s %(asctime)s %(module)s:%(lineno)3d] %(message)s" |
| 747 | else: |
| 748 | level = logging.INFO |
| 749 | fmt = "%(message)s" |
| 750 | logging.basicConfig(stream=sys.stdout, level=level, format=fmt) |
| 751 | |
Googler | 73d4fc9 | 2015-07-30 20:39:46 +0000 | [diff] [blame] | 752 | start_type = FLAGS.start |
| 753 | if FLAGS.start_app and start_type == "no": |
| 754 | start_type = "cold" |
| 755 | |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 756 | IncrementalInstall( |
| 757 | adb_path=FLAGS.adb, |
| 758 | adb_jobs=FLAGS.adb_jobs, |
| 759 | execroot=FLAGS.execroot, |
| 760 | stub_datafile=FLAGS.stub_datafile, |
| 761 | output_marker=FLAGS.output_marker, |
Googler | 73d4fc9 | 2015-07-30 20:39:46 +0000 | [diff] [blame] | 762 | start_type=start_type, |
Lukacs Berki | ecbf242 | 2015-07-28 07:57:45 +0000 | [diff] [blame] | 763 | native_libs=FLAGS.native_lib, |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 764 | split_main_apk=FLAGS.split_main_apk, |
| 765 | split_apks=FLAGS.split_apk, |
| 766 | dexmanifest=FLAGS.dexmanifest, |
| 767 | apk=FLAGS.apk, |
| 768 | resource_apk=FLAGS.resource_apk, |
| 769 | user_home_dir=FLAGS.user_home_dir) |
| 770 | |
| 771 | |
| 772 | if __name__ == "__main__": |
| 773 | FLAGS(sys.argv) |
| 774 | # process any additional flags in --flagfile |
| 775 | if FLAGS.flagfile: |
| 776 | with open(FLAGS.flagfile) as flagsfile: |
Lukacs Berki | 682ad92 | 2015-06-11 08:01:07 +0000 | [diff] [blame] | 777 | FLAGS.Reset() |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 778 | FLAGS(sys.argv + [line.strip() for line in flagsfile.readlines()]) |
Lukacs Berki | 682ad92 | 2015-06-11 08:01:07 +0000 | [diff] [blame] | 779 | |
Alex Humesky | a4ecde6 | 2015-05-21 17:08:42 +0000 | [diff] [blame] | 780 | main() |