Automating UI traversal and overlay dismissal with Frida

Overview

Automated UI traversal is useful when testing or instrumenting iOS apps at runtime. A traversal script can press controls, move through screens, and report state back to a host process. The main complication is that system overlays can interrupt the target app and block further interaction.

This article describes a Frida-based approach that separates app traversal from SpringBoard overlay handling. The host process coordinates both scripts and invokes the SpringBoard overlay dismissal path only when a button press occurs.

Objectives

Automatically traverse the UI of a target iOS application by pressing buttons and navigating screens.
Detect and dismiss SpringBoard overlays, specifically SBRemoteTransientOverlayHostViewController.
Inject scripts dynamically only when needed, optimizing Frida’s resource usage.

Design constraints

Overlay detection in SpringBoard
- Checking for overlays before every UI interaction introduced race conditions and unnecessary work.
- The host now calls the SpringBoard dismissal script only after an app button press.
Main-thread execution
- Many UI operations must be performed on the main thread to avoid errors like Cannot be called with asCopy = NO on non-main thread.
- UI interactions are wrapped in ObjC.schedule(ObjC.mainQueue, function() {...}).
Script lifecycle
- The app traversal script and the SpringBoard dismissal script run in different processes.
- The Python host owns the sessions and calls exported functions only when needed.

Implementation details

The host script (`host.py`)

The host script:

Attaches to both the target application and SpringBoard.
Loads traverse_app.js into the target app.
Injects dismiss_overlay.js into SpringBoard only when a button is pressed.

import frida
import sys
import time

APP_BUNDLE = "App Store"  # Change as needed.
SPRINGBOARD = "SpringBoard"

springboard_dismiss_script = None  # Global for dismiss_overlay.js
app_script = None

def on_spring_message(message, data):
    print("[SpringBoard]", message)

def on_app_message(message, data):
    print("[HOST] App message received:")
    print(message)
    if message["type"] == "send":
        payload = message.get("payload")
        if isinstance(payload, dict) and payload.get("type") == "button-pressed":
            print("[HOST] Button pressed, injecting dismiss_overlay.js")
            if springboard_dismiss_script:
                try:
                    dismiss_result = springboard_dismiss_script.exports_sync.dismissoverlay()
                    print("[HOST] SpringBoard overlay dismiss result:", dismiss_result)
                except Exception as e:
                    print("[HOST] Error calling dismissoverlay:", e)


def load_script(session, filename):
    with open(filename, "r") as f:
        script_code = f.read()
    script = session.create_script(script_code)
    script.on("message", on_spring_message)
    script.load()
    return script


def main():
    global springboard_dismiss_script, app_script
    device = frida.get_usb_device()

    print("[*] Attaching to SpringBoard")
    spring_session = device.attach(SPRINGBOARD)
    print("[*] Attaching to target app:", APP_BUNDLE)
    app_session = device.attach(APP_BUNDLE)

    # Load SpringBoard dismiss script
    springboard_dismiss_script = load_script(spring_session, "dismiss_overlay.js")

    # Load the target app's traversal script
    with open("traverse_app.js", "r") as f:
        app_script_code = f.read()
    app_script = app_session.create_script(app_script_code)
    app_script.on("message", on_app_message)
    app_script.load()

    print("[*] All scripts loaded. Press Ctrl+C to exit.")
    sys.stdin.read()

if __name__ == "__main__":
    main()

The UI traversal script (`traverse_app.js`)

This script automatically clicks UI elements and notifies the host when a button press occurs.

if (ObjC.available) {

    var pressedElements = {};

    function debugLog(msg) {
        console.log("[DEBUG] " + msg);
    }

    function findAllClickableElements(view) {
        var elements = [];
        if (!view) return elements;
        if (view.isKindOfClass_(ObjC.classes.UIControl) ||
            view.isKindOfClass_(ObjC.classes.UITableViewCell) ||
            (ObjC.classes.UITabBarButton && view.isKindOfClass_(ObjC.classes.UITabBarButton))) {
            elements.push(view);
        }
        var subs = view.subviews();
        for (var i = 0; i < subs.count(); i++) {
            elements = elements.concat(findAllClickableElements(subs.objectAtIndex_(i)));
        }
        return elements;
    }

    function pressElement(element, cb) {
        ObjC.schedule(ObjC.mainQueue, function() {
            try {
                element.sendActionsForControlEvents_(64);
                send({ type: "button-pressed" });  // Notify the host
            } catch (e) {
                debugLog("Error pressing element: " + e);
            }
            cb();
        });
    }

    function traverseScreen(done) {
        var keyWindow = ObjC.classes.UIWindow.keyWindow();
        if (!keyWindow) return;
        var elements = findAllClickableElements(keyWindow);
        if (elements.length === 0) return;
        pressElement(elements[0], function() {
            setTimeout(done, 2000);
        });
    }

    traverseScreen(function() {
        debugLog("Traversal complete.");
    });

} else {
    console.log("Objective-C runtime is not available.");
}

The overlay dismissal script (`dismiss_overlay.js`)

(function() {
    if (!ObjC.available) return;

    function debugLog(msg) {
        console.log("[DEBUG] " + msg);
    }

    var UIWindowClass = ObjC.classes.UIWindow;
    var windows = UIWindowClass.allWindows();
    if (!windows) return;

    for (var i = 0; i < windows.count(); i++) {
        var win = windows.objectAtIndex_(i);
        var vc = win.rootViewController();
        if (vc && vc.$className.indexOf("SBRemoteTransientOverlayHostViewController") !== -1) {
            ObjC.schedule(ObjC.mainQueue, function() {
                try {
                    vc.dismiss();
                    debugLog("Overlay dismissed.");
                } catch (e) {
                    debugLog("Error dismissing overlay: " + e);
                }
            });
            return;
        }
    }
})();

Summary

The host process keeps traversal and overlay handling separate. The app script identifies and presses UI elements, while the SpringBoard script dismisses transient overlays when the host observes a button press. This keeps the workflow simple and limits SpringBoard interaction to the points where overlays are likely to matter.

A useful next step is recording manual UI press events and replaying the sequence through Frida. That would make the traversal more deterministic than pressing the first discovered control on each screen.