AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make ASAN setup of Android bots more fault tolerant

Browse Source 
Bug: skia:7975
Change-Id: I3f123337913870a921d1c09714873eaf2949e88f
Reviewed-on: https://skia-review.googlesource.com/129364
Commit-Queue: Kevin Lubick <kjlubick@google.com>
Reviewed-by: Ben Wagner <benjaminwagner@google.com>
This commit is contained in:
Kevin Lubick 2018-05-21 15:47:14 -04:00 committed by Skia Commit-Bot
parent b5f2897ab5
commit 0e42586036
3 changed files with 89 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
infra/bots/recipe_modules/flavor/examples/full.expected/Test-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-Android_ASAN.json
View File

@ -98,7 +98,7 @@
"cmd": [
"python",
"-u",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\nASAN_SETUP = sys.argv[2]\n\ndef wait_for_device():\n while True:\n time.sleep(5)\n print 'Waiting for device'\n subprocess.check_output([ADB, 'wait-for-device'])\n bit1 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'dev.bootcomplete'])\n bit2 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'sys.boot_completed'])\n if '1' in bit1 and '1' in bit2:\n print 'Device detected'\n break\n\nlog = subprocess.check_output([ADB, 'root'])\n# check for message like 'adbd cannot run as root in production builds'\nprint log\nif 'cannot' in log:\n raise Exception('adb root failed')\n\noutput = subprocess.check_output([ADB, 'disable-verity'])\nprint output\n\nif 'already disabled' not in output:\n print 'Rebooting device'\n subprocess.check_output([ADB, 'reboot'])\n wait_for_device()\n\n# ASAN setup script is idempotent, either it installs it or says it's installed\noutput = subprocess.check_output([ADB, 'wait-for-device'])\nprocess = subprocess.Popen([ASAN_SETUP], env={'ADB': ADB},\n stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\n# this also blocks until command finishes\n(stdout, stderr) = process.communicate()\nprint stdout\nprint 'Stderr: %s' % stderr\nif process.returncode:\n raise Exception('setup ASAN returned with non-zero exit code: %d' %\n process.returncode)\n\nif 'Please wait until the device restarts' in stdout:\n # Sleep because device does not reboot instantly\n time.sleep(30)\nwait_for_device()\n",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\nASAN_SETUP = sys.argv[2]\n\ndef wait_for_device():\n while True:\n time.sleep(5)\n print 'Waiting for device'\n subprocess.check_output([ADB, 'wait-for-device'])\n bit1 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'dev.bootcomplete'])\n bit2 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'sys.boot_completed'])\n if '1' in bit1 and '1' in bit2:\n print 'Device detected'\n break\n\nlog = subprocess.check_output([ADB, 'root'])\n# check for message like 'adbd cannot run as root in production builds'\nprint log\nif 'cannot' in log:\n raise Exception('adb root failed')\n\noutput = subprocess.check_output([ADB, 'disable-verity'])\nprint output\n\nif 'already disabled' not in output:\n print 'Rebooting device'\n subprocess.check_output([ADB, 'reboot'])\n wait_for_device()\n\ndef installASAN(revert=False):\n # ASAN setup script is idempotent, either it installs it or\n # says it's installed. Returns True on success, false otherwise.\n out = subprocess.check_output([ADB, 'wait-for-device'])\n print out\n cmd = [ASAN_SETUP]\n if revert:\n cmd = [ASAN_SETUP, '--revert']\n process = subprocess.Popen(cmd, env={'ADB': ADB},\n stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\n # this also blocks until command finishes\n (stdout, stderr) = process.communicate()\n print stdout\n print 'Stderr: %s' % stderr\n return process.returncode == 0\n\nif not installASAN():\n print 'Trying to revert the ASAN install and then re-install'\n # ASAN script sometimes has issues if it was interrupted or partially applied\n # Try reverting it, then re-enabling it\n if not installASAN(revert=True):\n raise Exception('reverting ASAN install failed')\n\n # Sleep because device does not reboot instantly\n time.sleep(10)\n\n if not installASAN():\n raise Exception('Tried twice to setup ASAN and failed.')\n\n# Sleep because device does not reboot instantly\ntime.sleep(10)\nwait_for_device()\n",
"/opt/infra-android/tools/adb",
"[START_DIR]/android_ndk_linux/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/6.0.2/bin/asan_device_setup"
],
@ -145,22 +145,38 @@
"@@@STEP_LOG_LINE@python.inline@ subprocess.check_output([ADB, 'reboot'])@@@",
"@@@STEP_LOG_LINE@python.inline@ wait_for_device()@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@# ASAN setup script is idempotent, either it installs it or says it's installed@@@",
"@@@STEP_LOG_LINE@python.inline@output = subprocess.check_output([ADB, 'wait-for-device'])@@@",
"@@@STEP_LOG_LINE@python.inline@process = subprocess.Popen([ASAN_SETUP], env={'ADB': ADB},@@@",
"@@@STEP_LOG_LINE@python.inline@ stdout=subprocess.PIPE, stderr=subprocess.PIPE)@@@",
"@@@STEP_LOG_LINE@python.inline@def installASAN(revert=False):@@@",
"@@@STEP_LOG_LINE@python.inline@ # ASAN setup script is idempotent, either it installs it or@@@",
"@@@STEP_LOG_LINE@python.inline@ # says it's installed. Returns True on success, false otherwise.@@@",
"@@@STEP_LOG_LINE@python.inline@ out = subprocess.check_output([ADB, 'wait-for-device'])@@@",
"@@@STEP_LOG_LINE@python.inline@ print out@@@",
"@@@STEP_LOG_LINE@python.inline@ cmd = [ASAN_SETUP]@@@",
"@@@STEP_LOG_LINE@python.inline@ if revert:@@@",
"@@@STEP_LOG_LINE@python.inline@ cmd = [ASAN_SETUP, '--revert']@@@",
"@@@STEP_LOG_LINE@python.inline@ process = subprocess.Popen(cmd, env={'ADB': ADB},@@@",
"@@@STEP_LOG_LINE@python.inline@ stdout=subprocess.PIPE, stderr=subprocess.PIPE)@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@# this also blocks until command finishes@@@",
"@@@STEP_LOG_LINE@python.inline@(stdout, stderr) = process.communicate()@@@",
"@@@STEP_LOG_LINE@python.inline@print stdout@@@",
"@@@STEP_LOG_LINE@python.inline@print 'Stderr: %s' % stderr@@@",
"@@@STEP_LOG_LINE@python.inline@if process.returncode:@@@",
"@@@STEP_LOG_LINE@python.inline@ raise Exception('setup ASAN returned with non-zero exit code: %d' %@@@",
"@@@STEP_LOG_LINE@python.inline@ process.returncode)@@@",
"@@@STEP_LOG_LINE@python.inline@ # this also blocks until command finishes@@@",
"@@@STEP_LOG_LINE@python.inline@ (stdout, stderr) = process.communicate()@@@",
"@@@STEP_LOG_LINE@python.inline@ print stdout@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Stderr: %s' % stderr@@@",
"@@@STEP_LOG_LINE@python.inline@ return process.returncode == 0@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@if not installASAN():@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Trying to revert the ASAN install and then re-install'@@@",
"@@@STEP_LOG_LINE@python.inline@ # ASAN script sometimes has issues if it was interrupted or partially applied@@@",
"@@@STEP_LOG_LINE@python.inline@ # Try reverting it, then re-enabling it@@@",
"@@@STEP_LOG_LINE@python.inline@ if not installASAN(revert=True):@@@",
"@@@STEP_LOG_LINE@python.inline@ raise Exception('reverting ASAN install failed')@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@if 'Please wait until the device restarts' in stdout:@@@",
"@@@STEP_LOG_LINE@python.inline@ # Sleep because device does not reboot instantly@@@",
"@@@STEP_LOG_LINE@python.inline@ time.sleep(30)@@@",
"@@@STEP_LOG_LINE@python.inline@ time.sleep(10)@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@ if not installASAN():@@@",
"@@@STEP_LOG_LINE@python.inline@ raise Exception('Tried twice to setup ASAN and failed.')@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@# Sleep because device does not reboot instantly@@@",
"@@@STEP_LOG_LINE@python.inline@time.sleep(10)@@@",
"@@@STEP_LOG_LINE@python.inline@wait_for_device()@@@",
"@@@STEP_LOG_END@python.inline@@@"
]

42
infra/bots/recipe_modules/flavor/gn_android_flavor.py
View File

@ -389,22 +389,38 @@ if 'already disabled' not in output:
subprocess.check_output([ADB, 'reboot'])
wait_for_device()
# ASAN setup script is idempotent, either it installs it or says it's installed
output = subprocess.check_output([ADB, 'wait-for-device'])
process = subprocess.Popen([ASAN_SETUP], env={'ADB': ADB},
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def installASAN(revert=False):
# ASAN setup script is idempotent, either it installs it or
# says it's installed. Returns True on success, false otherwise.
out = subprocess.check_output([ADB, 'wait-for-device'])
print out
cmd = [ASAN_SETUP]
if revert:
cmd = [ASAN_SETUP, '--revert']
process = subprocess.Popen(cmd, env={'ADB': ADB},
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# this also blocks until command finishes
(stdout, stderr) = process.communicate()
print stdout
print 'Stderr: %s' % stderr
if process.returncode:
raise Exception('setup ASAN returned with non-zero exit code: %d' %
process.returncode)
# this also blocks until command finishes
(stdout, stderr) = process.communicate()
print stdout
print 'Stderr: %s' % stderr
return process.returncode == 0
if not installASAN():
print 'Trying to revert the ASAN install and then re-install'
# ASAN script sometimes has issues if it was interrupted or partially applied
# Try reverting it, then re-enabling it
if not installASAN(revert=True):
raise Exception('reverting ASAN install failed')
if 'Please wait until the device restarts' in stdout:
# Sleep because device does not reboot instantly
time.sleep(30)
time.sleep(10)
if not installASAN():
raise Exception('Tried twice to setup ASAN and failed.')
# Sleep because device does not reboot instantly
time.sleep(10)
wait_for_device()
""",
args = [self.ADB_BINARY, asan_setup],

44
infra/bots/recipes/test.expected/Test-Android-Clang-Nexus5x-GPU-Adreno418-arm-Debug-All-Android_ASAN.json
View File

@ -35,7 +35,7 @@
"cmd": [
"python",
"-u",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\nASAN_SETUP = sys.argv[2]\n\ndef wait_for_device():\n while True:\n time.sleep(5)\n print 'Waiting for device'\n subprocess.check_output([ADB, 'wait-for-device'])\n bit1 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'dev.bootcomplete'])\n bit2 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'sys.boot_completed'])\n if '1' in bit1 and '1' in bit2:\n print 'Device detected'\n break\n\nlog = subprocess.check_output([ADB, 'root'])\n# check for message like 'adbd cannot run as root in production builds'\nprint log\nif 'cannot' in log:\n raise Exception('adb root failed')\n\noutput = subprocess.check_output([ADB, 'disable-verity'])\nprint output\n\nif 'already disabled' not in output:\n print 'Rebooting device'\n subprocess.check_output([ADB, 'reboot'])\n wait_for_device()\n\n# ASAN setup script is idempotent, either it installs it or says it's installed\noutput = subprocess.check_output([ADB, 'wait-for-device'])\nprocess = subprocess.Popen([ASAN_SETUP], env={'ADB': ADB},\n stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\n# this also blocks until command finishes\n(stdout, stderr) = process.communicate()\nprint stdout\nprint 'Stderr: %s' % stderr\nif process.returncode:\n raise Exception('setup ASAN returned with non-zero exit code: %d' %\n process.returncode)\n\nif 'Please wait until the device restarts' in stdout:\n # Sleep because device does not reboot instantly\n time.sleep(30)\nwait_for_device()\n",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\nASAN_SETUP = sys.argv[2]\n\ndef wait_for_device():\n while True:\n time.sleep(5)\n print 'Waiting for device'\n subprocess.check_output([ADB, 'wait-for-device'])\n bit1 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'dev.bootcomplete'])\n bit2 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'sys.boot_completed'])\n if '1' in bit1 and '1' in bit2:\n print 'Device detected'\n break\n\nlog = subprocess.check_output([ADB, 'root'])\n# check for message like 'adbd cannot run as root in production builds'\nprint log\nif 'cannot' in log:\n raise Exception('adb root failed')\n\noutput = subprocess.check_output([ADB, 'disable-verity'])\nprint output\n\nif 'already disabled' not in output:\n print 'Rebooting device'\n subprocess.check_output([ADB, 'reboot'])\n wait_for_device()\n\ndef installASAN(revert=False):\n # ASAN setup script is idempotent, either it installs it or\n # says it's installed. Returns True on success, false otherwise.\n out = subprocess.check_output([ADB, 'wait-for-device'])\n print out\n cmd = [ASAN_SETUP]\n if revert:\n cmd = [ASAN_SETUP, '--revert']\n process = subprocess.Popen(cmd, env={'ADB': ADB},\n stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\n # this also blocks until command finishes\n (stdout, stderr) = process.communicate()\n print stdout\n print 'Stderr: %s' % stderr\n return process.returncode == 0\n\nif not installASAN():\n print 'Trying to revert the ASAN install and then re-install'\n # ASAN script sometimes has issues if it was interrupted or partially applied\n # Try reverting it, then re-enabling it\n if not installASAN(revert=True):\n raise Exception('reverting ASAN install failed')\n\n # Sleep because device does not reboot instantly\n time.sleep(10)\n\n if not installASAN():\n raise Exception('Tried twice to setup ASAN and failed.')\n\n# Sleep because device does not reboot instantly\ntime.sleep(10)\nwait_for_device()\n",
"/opt/infra-android/tools/adb",
"[START_DIR]/android_ndk_linux/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/6.0.2/bin/asan_device_setup"
],
@ -82,22 +82,38 @@
"@@@STEP_LOG_LINE@python.inline@ subprocess.check_output([ADB, 'reboot'])@@@",
"@@@STEP_LOG_LINE@python.inline@ wait_for_device()@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@# ASAN setup script is idempotent, either it installs it or says it's installed@@@",
"@@@STEP_LOG_LINE@python.inline@output = subprocess.check_output([ADB, 'wait-for-device'])@@@",
"@@@STEP_LOG_LINE@python.inline@process = subprocess.Popen([ASAN_SETUP], env={'ADB': ADB},@@@",
"@@@STEP_LOG_LINE@python.inline@ stdout=subprocess.PIPE, stderr=subprocess.PIPE)@@@",
"@@@STEP_LOG_LINE@python.inline@def installASAN(revert=False):@@@",
"@@@STEP_LOG_LINE@python.inline@ # ASAN setup script is idempotent, either it installs it or@@@",
"@@@STEP_LOG_LINE@python.inline@ # says it's installed. Returns True on success, false otherwise.@@@",
"@@@STEP_LOG_LINE@python.inline@ out = subprocess.check_output([ADB, 'wait-for-device'])@@@",
"@@@STEP_LOG_LINE@python.inline@ print out@@@",
"@@@STEP_LOG_LINE@python.inline@ cmd = [ASAN_SETUP]@@@",
"@@@STEP_LOG_LINE@python.inline@ if revert:@@@",
"@@@STEP_LOG_LINE@python.inline@ cmd = [ASAN_SETUP, '--revert']@@@",
"@@@STEP_LOG_LINE@python.inline@ process = subprocess.Popen(cmd, env={'ADB': ADB},@@@",
"@@@STEP_LOG_LINE@python.inline@ stdout=subprocess.PIPE, stderr=subprocess.PIPE)@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@# this also blocks until command finishes@@@",
"@@@STEP_LOG_LINE@python.inline@(stdout, stderr) = process.communicate()@@@",
"@@@STEP_LOG_LINE@python.inline@print stdout@@@",
"@@@STEP_LOG_LINE@python.inline@print 'Stderr: %s' % stderr@@@",
"@@@STEP_LOG_LINE@python.inline@if process.returncode:@@@",
"@@@STEP_LOG_LINE@python.inline@ raise Exception('setup ASAN returned with non-zero exit code: %d' %@@@",
"@@@STEP_LOG_LINE@python.inline@ process.returncode)@@@",
"@@@STEP_LOG_LINE@python.inline@ # this also blocks until command finishes@@@",
"@@@STEP_LOG_LINE@python.inline@ (stdout, stderr) = process.communicate()@@@",
"@@@STEP_LOG_LINE@python.inline@ print stdout@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Stderr: %s' % stderr@@@",
"@@@STEP_LOG_LINE@python.inline@ return process.returncode == 0@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@if not installASAN():@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Trying to revert the ASAN install and then re-install'@@@",
"@@@STEP_LOG_LINE@python.inline@ # ASAN script sometimes has issues if it was interrupted or partially applied@@@",
"@@@STEP_LOG_LINE@python.inline@ # Try reverting it, then re-enabling it@@@",
"@@@STEP_LOG_LINE@python.inline@ if not installASAN(revert=True):@@@",
"@@@STEP_LOG_LINE@python.inline@ raise Exception('reverting ASAN install failed')@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@if 'Please wait until the device restarts' in stdout:@@@",
"@@@STEP_LOG_LINE@python.inline@ # Sleep because device does not reboot instantly@@@",
"@@@STEP_LOG_LINE@python.inline@ time.sleep(30)@@@",
"@@@STEP_LOG_LINE@python.inline@ time.sleep(10)@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@ if not installASAN():@@@",
"@@@STEP_LOG_LINE@python.inline@ raise Exception('Tried twice to setup ASAN and failed.')@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@# Sleep because device does not reboot instantly@@@",
"@@@STEP_LOG_LINE@python.inline@time.sleep(10)@@@",
"@@@STEP_LOG_LINE@python.inline@wait_for_device()@@@",
"@@@STEP_LOG_END@python.inline@@@"
]