AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Try a workaround for the Pixel ASAN push failure

Browse Source 
There have been many recent instances where ASAN tasks running on Pixel
devices have failed to push resources to the device due to
"secure_mkdirs failed: No such file or directory." The theory is that
because the "Setting up device to run ASAN" step reboots the device, the
device isn't actually ready at the time it reports being ready. This CL
adds a sleep to the ASAN setup to work around this problem.

Here are recent examples of this failure:
https://chromium-swarm.appspot.com/task?id=4279aeb78961de10
https://chromium-swarm.appspot.com/task?id=42765d9f44ca4610
https://chromium-swarm.appspot.com/task?id=427a66499846e910
https://chromium-swarm.appspot.com/task?id=4276d85a8f270d10
https://chromium-swarm.appspot.com/task?id=42765cb3ff062a10

Change-Id: Iddf78622de3cd88ea28a532380762baafe479eb7
Reviewed-on: https://skia-review.googlesource.com/c/185320
Auto-Submit: Ben Wagner <benjaminwagner@google.com>
Reviewed-by: Kevin Lubick <kjlubick@google.com>
Commit-Queue: Ben Wagner <benjaminwagner@google.com>
This commit is contained in:
Ben Wagner 2019-01-18 13:07:07 -05:00 committed by Skia Commit-Bot
parent 6f6ae6a59f
commit 64b894b98c
3 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
infra/bots/recipe_modules/flavor/android.py
View File

@ -423,12 +423,16 @@ if not installASAN():
# Sleep because device does not reboot instantly
time.sleep(10)
wait_for_device()
# Sleep again to hopefully avoid error "secure_mkdirs failed: No such file or
# directory" when pushing resources to the device.
time.sleep(20)
""",
args = [self.ADB_BINARY, asan_setup],
infra_step=True,
timeout=300,
abort_on_failure=True)
def cleanup_steps(self):
if 'ASAN' in self.m.vars.extra_tokens:
self._ever_ran_adb = True

5
infra/bots/recipe_modules/flavor/examples/full.expected/Test-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-Android_ASAN.json
View File

@ -112,7 +112,7 @@
"cmd": [
"python",
"-u",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\nASAN_SETUP = sys.argv[2]\n\ndef wait_for_device():\n while True:\n time.sleep(5)\n print 'Waiting for device'\n subprocess.check_output([ADB, 'wait-for-device'])\n bit1 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'dev.bootcomplete'])\n bit2 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'sys.boot_completed'])\n if '1' in bit1 and '1' in bit2:\n print 'Device detected'\n break\n\nlog = subprocess.check_output([ADB, 'root'])\n# check for message like 'adbd cannot run as root in production builds'\nprint log\nif 'cannot' in log:\n raise Exception('adb root failed')\n\noutput = subprocess.check_output([ADB, 'disable-verity'])\nprint output\n\nif 'already disabled' not in output:\n print 'Rebooting device'\n subprocess.check_output([ADB, 'reboot'])\n wait_for_device()\n\ndef installASAN(revert=False):\n # ASAN setup script is idempotent, either it installs it or\n # says it's installed. Returns True on success, false otherwise.\n out = subprocess.check_output([ADB, 'wait-for-device'])\n print out\n cmd = [ASAN_SETUP]\n if revert:\n cmd = [ASAN_SETUP, '--revert']\n process = subprocess.Popen(cmd, env={'ADB': ADB},\n stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\n # this also blocks until command finishes\n (stdout, stderr) = process.communicate()\n print stdout\n print 'Stderr: %s' % stderr\n return process.returncode == 0\n\nif not installASAN():\n print 'Trying to revert the ASAN install and then re-install'\n # ASAN script sometimes has issues if it was interrupted or partially applied\n # Try reverting it, then re-enabling it\n if not installASAN(revert=True):\n raise Exception('reverting ASAN install failed')\n\n # Sleep because device does not reboot instantly\n time.sleep(10)\n\n if not installASAN():\n raise Exception('Tried twice to setup ASAN and failed.')\n\n# Sleep because device does not reboot instantly\ntime.sleep(10)\nwait_for_device()\n",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\nASAN_SETUP = sys.argv[2]\n\ndef wait_for_device():\n while True:\n time.sleep(5)\n print 'Waiting for device'\n subprocess.check_output([ADB, 'wait-for-device'])\n bit1 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'dev.bootcomplete'])\n bit2 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'sys.boot_completed'])\n if '1' in bit1 and '1' in bit2:\n print 'Device detected'\n break\n\nlog = subprocess.check_output([ADB, 'root'])\n# check for message like 'adbd cannot run as root in production builds'\nprint log\nif 'cannot' in log:\n raise Exception('adb root failed')\n\noutput = subprocess.check_output([ADB, 'disable-verity'])\nprint output\n\nif 'already disabled' not in output:\n print 'Rebooting device'\n subprocess.check_output([ADB, 'reboot'])\n wait_for_device()\n\ndef installASAN(revert=False):\n # ASAN setup script is idempotent, either it installs it or\n # says it's installed. Returns True on success, false otherwise.\n out = subprocess.check_output([ADB, 'wait-for-device'])\n print out\n cmd = [ASAN_SETUP]\n if revert:\n cmd = [ASAN_SETUP, '--revert']\n process = subprocess.Popen(cmd, env={'ADB': ADB},\n stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\n # this also blocks until command finishes\n (stdout, stderr) = process.communicate()\n print stdout\n print 'Stderr: %s' % stderr\n return process.returncode == 0\n\nif not installASAN():\n print 'Trying to revert the ASAN install and then re-install'\n # ASAN script sometimes has issues if it was interrupted or partially applied\n # Try reverting it, then re-enabling it\n if not installASAN(revert=True):\n raise Exception('reverting ASAN install failed')\n\n # Sleep because device does not reboot instantly\n time.sleep(10)\n\n if not installASAN():\n raise Exception('Tried twice to setup ASAN and failed.')\n\n# Sleep because device does not reboot instantly\ntime.sleep(10)\nwait_for_device()\n# Sleep again to hopefully avoid error \"secure_mkdirs failed: No such file or\n# directory\" when pushing resources to the device.\ntime.sleep(20)\n",
"/opt/infra-android/tools/adb",
"[START_DIR]/android_ndk_linux/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/8.0.2/bin/asan_device_setup"
],
@ -192,6 +192,9 @@
"@@@STEP_LOG_LINE@python.inline@# Sleep because device does not reboot instantly@@@",
"@@@STEP_LOG_LINE@python.inline@time.sleep(10)@@@",
"@@@STEP_LOG_LINE@python.inline@wait_for_device()@@@",
"@@@STEP_LOG_LINE@python.inline@# Sleep again to hopefully avoid error \"secure_mkdirs failed: No such file or@@@",
"@@@STEP_LOG_LINE@python.inline@# directory\" when pushing resources to the device.@@@",
"@@@STEP_LOG_LINE@python.inline@time.sleep(20)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},

5
infra/bots/recipes/test.expected/Test-Android-Clang-Pixel-GPU-Adreno530-arm-Debug-All-Android_ASAN.json
View File

@ -49,7 +49,7 @@
"cmd": [
"python",
"-u",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\nASAN_SETUP = sys.argv[2]\n\ndef wait_for_device():\n while True:\n time.sleep(5)\n print 'Waiting for device'\n subprocess.check_output([ADB, 'wait-for-device'])\n bit1 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'dev.bootcomplete'])\n bit2 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'sys.boot_completed'])\n if '1' in bit1 and '1' in bit2:\n print 'Device detected'\n break\n\nlog = subprocess.check_output([ADB, 'root'])\n# check for message like 'adbd cannot run as root in production builds'\nprint log\nif 'cannot' in log:\n raise Exception('adb root failed')\n\noutput = subprocess.check_output([ADB, 'disable-verity'])\nprint output\n\nif 'already disabled' not in output:\n print 'Rebooting device'\n subprocess.check_output([ADB, 'reboot'])\n wait_for_device()\n\ndef installASAN(revert=False):\n # ASAN setup script is idempotent, either it installs it or\n # says it's installed. Returns True on success, false otherwise.\n out = subprocess.check_output([ADB, 'wait-for-device'])\n print out\n cmd = [ASAN_SETUP]\n if revert:\n cmd = [ASAN_SETUP, '--revert']\n process = subprocess.Popen(cmd, env={'ADB': ADB},\n stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\n # this also blocks until command finishes\n (stdout, stderr) = process.communicate()\n print stdout\n print 'Stderr: %s' % stderr\n return process.returncode == 0\n\nif not installASAN():\n print 'Trying to revert the ASAN install and then re-install'\n # ASAN script sometimes has issues if it was interrupted or partially applied\n # Try reverting it, then re-enabling it\n if not installASAN(revert=True):\n raise Exception('reverting ASAN install failed')\n\n # Sleep because device does not reboot instantly\n time.sleep(10)\n\n if not installASAN():\n raise Exception('Tried twice to setup ASAN and failed.')\n\n# Sleep because device does not reboot instantly\ntime.sleep(10)\nwait_for_device()\n",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\nASAN_SETUP = sys.argv[2]\n\ndef wait_for_device():\n while True:\n time.sleep(5)\n print 'Waiting for device'\n subprocess.check_output([ADB, 'wait-for-device'])\n bit1 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'dev.bootcomplete'])\n bit2 = subprocess.check_output([ADB, 'shell', 'getprop',\n 'sys.boot_completed'])\n if '1' in bit1 and '1' in bit2:\n print 'Device detected'\n break\n\nlog = subprocess.check_output([ADB, 'root'])\n# check for message like 'adbd cannot run as root in production builds'\nprint log\nif 'cannot' in log:\n raise Exception('adb root failed')\n\noutput = subprocess.check_output([ADB, 'disable-verity'])\nprint output\n\nif 'already disabled' not in output:\n print 'Rebooting device'\n subprocess.check_output([ADB, 'reboot'])\n wait_for_device()\n\ndef installASAN(revert=False):\n # ASAN setup script is idempotent, either it installs it or\n # says it's installed. Returns True on success, false otherwise.\n out = subprocess.check_output([ADB, 'wait-for-device'])\n print out\n cmd = [ASAN_SETUP]\n if revert:\n cmd = [ASAN_SETUP, '--revert']\n process = subprocess.Popen(cmd, env={'ADB': ADB},\n stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\n # this also blocks until command finishes\n (stdout, stderr) = process.communicate()\n print stdout\n print 'Stderr: %s' % stderr\n return process.returncode == 0\n\nif not installASAN():\n print 'Trying to revert the ASAN install and then re-install'\n # ASAN script sometimes has issues if it was interrupted or partially applied\n # Try reverting it, then re-enabling it\n if not installASAN(revert=True):\n raise Exception('reverting ASAN install failed')\n\n # Sleep because device does not reboot instantly\n time.sleep(10)\n\n if not installASAN():\n raise Exception('Tried twice to setup ASAN and failed.')\n\n# Sleep because device does not reboot instantly\ntime.sleep(10)\nwait_for_device()\n# Sleep again to hopefully avoid error \"secure_mkdirs failed: No such file or\n# directory\" when pushing resources to the device.\ntime.sleep(20)\n",
"/usr/bin/adb.1.0.35",
"[START_DIR]/android_ndk_linux/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/8.0.2/bin/asan_device_setup"
],
@ -129,6 +129,9 @@
"@@@STEP_LOG_LINE@python.inline@# Sleep because device does not reboot instantly@@@",
"@@@STEP_LOG_LINE@python.inline@time.sleep(10)@@@",
"@@@STEP_LOG_LINE@python.inline@wait_for_device()@@@",
"@@@STEP_LOG_LINE@python.inline@# Sleep again to hopefully avoid error \"secure_mkdirs failed: No such file or@@@",
"@@@STEP_LOG_LINE@python.inline@# directory\" when pushing resources to the device.@@@",
"@@@STEP_LOG_LINE@python.inline@time.sleep(20)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},