[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10,"_11":12},"actionData","errors","root",{"_164":-5,"_167":168,"_169":170},"routes/shell",{"_40":41,"_163":-5,"_164":-5,"_165":166},"routes/repo",{"_33":-7,"_34":35,"_36":37,"_38":39,"_40":41,"_42":31,"_43":31,"_44":31,"_45":31,"_46":31,"_47":31,"_48":49,"_50":49,"_51":31,"_52":-7,"_53":-7,"_54":55,"_56":57,"_58":59,"_60":-5,"_61":62,"_63":64,"_65":66,"_67":68},"routes/repo.blob",{"_13":14,"_15":16,"_17":18,"_19":20,"_21":14,"_22":23,"_24":-5,"_25":26,"_27":28,"_28":29,"_30":31,"_32":14},"ref","c2a499a3cbcc261b05d68f2218387ea98ecc96f7","path","native/sources/client/client.swift","oid","e1971d87da7ff4224daff89be63dc2e3f8c61094","size",14214,"commit","mounts",[],"source","plain",{"_13":14,"_15":16},"kind","text","import DesktopToolsIdentity\nimport Darwin\nimport Foundation\nimport PeekabooAutomationKit\nimport PeekabooBridge\n\n@main\nprivate enum Client {\n\tstatic func main() async {\n\t\tdo {\n\t\t\twrite(try await execute(Array(CommandLine.arguments.dropFirst())))\n\t\t} catch ClientError.dialect {\n\t\t\twrite(try! JSONEncoder().encode(Failure(error: Message(\n\t\t\t\tcode: \"DESKTOP_PROTOCOL_MISMATCH\", message: ClientError.dialect.localizedDescription\n\t\t\t))))\n\t\t\texit(1)\n\t\t} catch let error as PeekabooBridgeErrorEnvelope {\n\t\t\twrite(try! JSONEncoder().encode(Failure(error: error)))\n\t\t\texit(1)\n\t\t} catch {\n\t\t\twrite(try! JSONEncoder().encode(Failure(error: Message(\n\t\t\t\tcode: \"DESKTOP_ERROR\", message: error.localizedDescription\n\t\t\t))))\n\t\t\texit(1)\n\t\t}\n\t}\n\n\tprivate static func execute(_ args: [String]) async throws -> Data {\n\t\tguard args.count >= 2 else { throw ClientError.usage }\n\t\tlet operation = args[1]\n\t\tguard (operation == \"apps\" && args.count == 2)\n\t\t\t|| (operation == \"windows\" && args.count == 3)\n\t\t\t|| (operation == \"inspect\" && (args.count == 5 || args.count == 6))\n\t\t\t|| (operation == \"action\" && args.count == 2)\n\t\t\t|| (operation == \"management\" && args.count == 2)\n\t\t\t|| (operation == \"launch\" && args.count == 2)\n\t\t\t|| (operation == \"open\" && args.count == 2)\n\t\t\t|| (operation == \"clipboard\" && args.count == 2)\n\t\t\t|| (operation == \"menus\" && args.count == 2)\n\t\t\t|| (operation == \"menu\" && args.count == 2)\n\t\telse { throw ClientError.usage }\n\t\tlet identity = try SigningIdentity.current()\n\t\tlet client = PeekabooBridgeClient(\n\t\t\tsocketPath: args[0],\n\t\t\tmaxResponseBytes: 48 * 1024 * 1024,\n\t\t\trequestTimeoutSec: 25,\n\t\t\ttrustedHostTeamIDs: [identity.team]\n\t\t)\n\t\tlet handshake = try await client.handshake(\n\t\t\tclient: .init(\n\t\t\t\tbundleIdentifier: identity.identifier,\n\t\t\t\tteamIdentifier: identity.team,\n\t\t\t\tprocessIdentifier: getpid()\n\t\t\t),\n\t\t\trequestedHost: .gui,\n\t\t\toverallTimeoutSec: 5\n\t\t)\n\t\t// Patched operations reuse Peekaboo's protocol version; refuse other hosts before any request.\n\t\tguard handshake.hostCapabilities?.contains(desktopToolsProtocol) == true else { throw ClientError.dialect }\n\t\tswitch operation {\n\t\tcase \"action\":\n\t\t\treturn try await nativeAction(client)\n\t\tcase \"clipboard\":\n\t\t\treturn try await nativeClipboard(client)\n\t\tcase \"menu\":\n\t\t\treturn try await nativeMenuCommand(client, handshake: handshake)\n\t\tcase \"menus\":\n\t\t\treturn try await nativeMenus(client)\n\t\tcase \"management\":\n\t\t\treturn try await nativeManagement(client, handshake: handshake)\n\t\tcase \"launch\":\n\t\t\treturn try await nativeLaunch(client, handshake: handshake)\n\t\tcase \"open\":\n\t\t\treturn try await nativeLaunch(client, handshake: handshake, opensItem: true)\n\t\tcase \"apps\":\n\t\t\tlet inventory = try await client.listApplicationMutationInventory()\n\t\t\tvar metadata: [ServiceApplicationInfo] = []\n\t\t\tvar warnings: [String] = []\n\t\t\tdo {\n\t\t\t\tmetadata = try await client.listApplications()\n\t\t\t} catch {\n\t\t\t\ttry Task.checkCancellation()\n\t\t\t\twarnings.append(\"Application presentation metadata was unavailable; activity and visibility are unknown.\")\n\t\t\t}\n\t\t\tlet grouped = Dictionary(grouping: metadata, by: \\.processIdentifier)\n\t\t\tvar matched: [Int32: ServiceApplicationInfo] = [:]\n\t\t\tfor app in inventory.items {\n\t\t\t\tguard let generation = app.processStartIdentity, generation > 0 else { continue }\n\t\t\t\tlet candidates = (grouped[app.processIdentifier] ?? []).filter { $0.processStartIdentity == generation }\n\t\t\t\tif candidates.count == 1 { matched[app.processIdentifier] = candidates[0] }\n\t\t\t}\n\t\t\tlet active = metadata.filter(\\.isActive)\n\t\t\tlet activityKnown = active.count == 1\n\t\t\t\t&& active[0].processStartIdentity != nil\n\t\t\t\t&& matched[active[0].processIdentifier]?.processStartIdentity == active[0].processStartIdentity\n\t\t\tif !activityKnown {\n\t\t\t\twarnings.append(\"No unique active application with matching process-generation identity was observed; activity is unknown.\")\n\t\t\t}\n\t\t\tif matched.count < inventory.items.count {\n\t\t\t\twarnings.append(\"Some applications lacked presentation metadata matching their process generation; their activity and visibility are unknown.\")\n\t\t\t}\n\t\t\treturn try encode(Apps(\n\t\t\t\tapps: inventory.items.map { App($0, metadata: matched[$0.processIdentifier], activityKnown: activityKnown) },\n\t\t\t\tinventory_completeness: inventory.completeness.rawValue,\n\t\t\t\tinventory_warnings: inventory.warnings,\n\t\t\t\tmetadata_warnings: warnings\n\t\t\t))\n\t\tcase \"windows\":\n\t\t\tlet pid = try processID(args[2])\n\t\t\tlet inventory = try await client.listWindowMutationInventory(target: .application(\"PID:\\(pid)\"))\n\t\t\treturn try encode(Windows(\n\t\t\t\tpid: pid,\n\t\t\t\twindows: inventory.items.map { Window($0, pid: pid) },\n\t\t\t\tinventory_completeness: inventory.completeness.rawValue,\n\t\t\t\tinventory_warnings: inventory.warnings\n\t\t\t))\n\t\tdefault:\n\t\t\tlet pid = try processID(args[2])\n\t\t\tguard let window = UInt32(args[3]), window > 0 else { throw ClientError.usage }\n\t\t\tguard args[4].hasPrefix(\"/\") else { throw ClientError.usage }\n\t\t\tguard let mode = InspectionMode(rawValue: args.count == 6 ? args[5] : \"accessibility\")\n\t\t\telse { throw ClientError.usage }\n\t\t\treturn try await inspect(client, pid: pid, window: window, path: args[4], mode: mode)\n\t\t}\n\t}\n\n\tprivate static func inspect(\n\t\t_ client: PeekabooBridgeClient, pid: Int32, window: UInt32, path: String, mode: InspectionMode\n\t) async throws -> Data {\n\t\tguard mode == .pixels else { return try await observe(client, pid: pid, window: window, path: path, pixels: nil) }\n\t\t// Screenshot-only results have no detection ID, so the request pins its publication explicitly.\n\t\tlet snapshot = try await client.createSnapshot()\n\t\tdo {\n\t\t\treturn try await observe(client, pid: pid, window: window, path: path, pixels: snapshot)\n\t\t} catch {\n\t\t\ttry? await client.cleanSnapshot(snapshotId: snapshot)\n\t\t\tthrow error\n\t\t}\n\t}\n\n\tprivate static func observe(\n\t\t_ client: PeekabooBridgeClient, pid: Int32, window: UInt32, path: String, pixels: String?\n\t) async throws -> Data {\n\t\tlet accessibility = pixels == nil\n\t\tlet observation = try await client.desktopObservationWithOutcome(.init(\n\t\t\ttarget: .pid(pid, window: .id(window)),\n\t\t\tcapture: .init(scale: .logical1x, focus: .background),\n\t\t\tdetection: .init(\n\t\t\t\tmode: accessibility ? .accessibility : .none,\n\t\t\t\ttraversalBudget: .init(maxDepth: 15, maxElementCount: 200, maxChildrenPerNode: 100),\n\t\t\t\trequiresFreshAccessibilityTree: accessibility\n\t\t\t),\n\t\t\toutput: .init(path: path, saveSnapshot: true, snapshotID: pixels, includeImageData: true),\n\t\t\ttimeout: .init(overall: 20, detection: 15)\n\t\t))\n\t\tguard let target = observation.targetIdentity,\n\t\t\ttarget.processIdentity.processIdentifier == pid,\n\t\t\tlet exact = target.exactWindow, exact.identity.windowID == Int(window)\n\t\telse { throw ClientError.target }\n\t\tlet result = observation.payload\n\t\tif let pixels {\n\t\t\tguard result.files.publishedSnapshotID == pixels, result.elements == nil else { throw ClientError.pixels }\n\t\t\t// Expose the ID only after the stored canonical projection grants pointer authority for this exact capture.\n\t\t\tlet detection = try await client.getDetectionResult(snapshotId: pixels)\n\t\t\tlet (context, identity, bounds) = try snapshotWindow(detection)\n\t\t\tlet authority = try coordinateAuthority(\n\t\t\t\tpixels, detection, window: .init(identity: identity, bounds: bounds)\n\t\t\t)\n\t\t\tguard detection.snapshotId == pixels, detection.elements.all.isEmpty, context.focusedElement == nil,\n\t\t\t\tidentity.hasSameStableReceipt(as: exact.identity), bounds == exact.bounds,\n\t\t\t\tauthority.context == CaptureCoordinateContext(metadata: result.capture.metadata, referenceID: pixels)\n\t\t\telse { throw ClientError.pixels }\n\t\t}\n\t\tlet image = try result.verifiedCaptureImageData(requirement: .requireDigest)\n\t\tguard !image.isEmpty, image.count <= 32 * 1024 * 1024 else { throw ClientError.image }\n\t\t// Reusable snapshots keep Bridge-owned artifact copies; the host removes this caller-visible file after resizing.\n\t\ttry image.write(to: URL(fileURLWithPath: path), options: [.atomic])\n\t\tlet data = Inspection(\n\t\t\tinspection_mode: accessibility ? InspectionMode.accessibility.rawValue : InspectionMode.pixels.rawValue,\n\t\t\tnote: accessibility ? nil : \"Pixel snapshot: no Accessibility elements or focused control. Its snapshot_id authorizes only screenshot-point clicks, point scrolls, and drags in this exact window; element, text, key, and insertion actions are refused.\",\n\t\t\tapplication_name: result.target.app?.name,\n\t\t\twindow_title: result.target.window?.title,\n\t\t\tsnapshot_id: result.files.publishedSnapshotID,\n\t\t\telement_count: result.elements?.metadata.elementCount ?? 0,\n\t\t\tui_elements: result.elements?.elements.all.map(Element.init) ?? [],\n\t\t\tcoordinate_context: result.elements?.metadata.captureCoordinateContext\n\t\t\t\t?? CaptureCoordinateContext(metadata: result.capture.metadata),\n\t\t\tcapture_warning: result.capture.warning,\n\t\t\tdetection_metadata: result.elements?.metadata,\n\t\t\tdiagnostics: result.diagnostics,\n\t\t\ttimings: result.timings\n\t\t)\n\t\treturn try encode(data, receipt: .init(\n\t\t\tpid: pid,\n\t\t\twindow_id: Int(window),\n\t\t\tprocess_start_identity_decimal: String(target.processIdentity.processStartIdentity)\n\t\t))\n\t}\n\n\tprivate static func processID(_ value: String) throws -> Int32 {\n\t\tguard let pid = Int32(value), pid > 0 else { throw ClientError.usage }\n\t\treturn pid\n\t}\n\n\tprivate static func encode(_ data: T, receipt: Receipt? = nil) throws -> Data {\n\t\ttry JSONEncoder().encode(Success(data: data, target_receipt: receipt))\n\t}\n\n\tprivate static func write(_ data: Data) {\n\t\tFileHandle.standardOutput.write(data)\n\t\tFileHandle.standardOutput.write(Data([10]))\n\t}\n}\n\nprivate struct Success: Encodable {\n\tlet success = true\n\tlet data: T\n\tlet target_receipt: Receipt?\n}\n\nprivate struct Failure: Encodable {\n\tlet success = false\n\tlet error: T\n}\n\nprivate struct Message: Encodable {\n\tlet code: String\n\tlet message: String\n}\n\nstruct Receipt: Encodable {\n\tlet pid: Int32\n\tlet window_id: Int?\n\tlet process_start_identity_decimal: String\n}\n\nprivate struct Apps: Encodable {\n\tlet apps: [App]\n\tlet inventory_completeness: String\n\tlet inventory_warnings: [String]\n\tlet metadata_warnings: [String]\n}\n\nprivate struct App: Encodable {\n\tlet name: String\n\tlet pid: Int32\n\tlet bundle_id: String?\n\tlet is_active: Bool?\n\tlet is_active_known: Bool\n\tlet is_hidden: Bool?\n\tlet is_hidden_known: Bool\n\tlet process_start_identity_decimal: String?\n\tlet warnings: [String]?\n\tlet target: ManagementTarget?\n\n\tinit(_ app: ServiceApplicationInfo, metadata: ServiceApplicationInfo?, activityKnown: Bool) {\n\t\tname = app.name\n\t\tpid = app.processIdentifier\n\t\tbundle_id = app.bundleIdentifier\n\t\t// Mutation inventory supplies identity, not presentation state. Missing reads must stay unknown.\n\t\tis_active_known = metadata != nil && activityKnown\n\t\tis_active = is_active_known ? metadata?.isActive : nil\n\t\tis_hidden_known = metadata?.isHiddenKnown == true\n\t\tis_hidden = is_hidden_known ? metadata?.isHidden : nil\n\t\tprocess_start_identity_decimal = app.processStartIdentity.map(String.init)\n\t\tlet combined = (app.metadataWarnings ?? []) + (metadata?.metadataWarnings ?? [])\n\t\twarnings = combined.isEmpty ? nil : Array(Set(combined)).sorted()\n\t\ttarget = app.processIdentity.map(ManagementTarget.init)\n\t}\n}\n\nprivate struct Windows: Encodable {\n\tlet pid: Int32\n\tlet windows: [Window]\n\tlet inventory_completeness: String\n\tlet inventory_warnings: [String]\n}\n\nprivate struct Window: Encodable {\n\tlet window_id: Int\n\tlet window_title: String\n\tlet bounds: Bounds\n\tlet is_on_screen: Bool\n\tlet is_minimized: Bool\n\tlet is_key: Bool?\n\tlet observation_capability: String?\n\tlet observation_reason: String?\n\tlet process_start_identity_decimal: String?\n\tlet target: ManagementTarget?\n\n\tinit(_ window: ServiceWindowInfo, pid: Int32) {\n\t\twindow_id = window.windowID\n\t\twindow_title = window.title\n\t\tbounds = Bounds(window.bounds)\n\t\tis_on_screen = window.isOnScreen\n\t\tis_minimized = window.isMinimized\n\t\tis_key = window.isKeyWindow\n\t\tobservation_capability = window.observationCapability?.mode.rawValue\n\t\tobservation_reason = window.observationCapability?.reason?.rawValue\n\t\tprocess_start_identity_decimal = window.mutationIdentity.map { String($0.processIdentity.processStartIdentity) }\n\t\ttarget = ManagementTarget(window: window, pid: pid)\n\t}\n}\n\nstruct Bounds: Codable {\n\tlet x: Double\n\tlet y: Double\n\tlet width: Double\n\tlet height: Double\n\n\tinit(_ rectangle: CGRect) {\n\t\tx = rectangle.origin.x\n\t\ty = rectangle.origin.y\n\t\twidth = rectangle.width\n\t\theight = rectangle.height\n\t}\n}\n\nprivate struct Inspection: Encodable {\n\tlet inspection_mode: String\n\tlet note: String?\n\tlet application_name: String?\n\tlet window_title: String?\n\tlet snapshot_id: String?\n\tlet element_count: Int\n\tlet ui_elements: [Element]\n\tlet coordinate_context: CaptureCoordinateContext\n\tlet capture_warning: String?\n\tlet detection_metadata: DetectionMetadata?\n\tlet diagnostics: DesktopObservationDiagnostics\n\tlet timings: ObservationTimings\n}\n\nprivate struct Element: Encodable {\n\tlet id: String\n\tlet role: String\n\tlet label: String?\n\tlet value: String?\n\tlet bounds: Bounds\n\tlet is_enabled: Bool\n\tlet is_selected: Bool?\n\tlet attributes: [String: String]\n\n\tinit(_ element: DetectedElement) {\n\t\tid = element.id\n\t\trole = element.type.rawValue\n\t\tlabel = element.label\n\t\tvalue = element.value\n\t\tbounds = Bounds(element.bounds)\n\t\tis_enabled = element.isEnabled\n\t\tis_selected = element.isSelected\n\t\tattributes = element.attributes\n\t}\n}\n\nprivate enum InspectionMode: String {\n\tcase accessibility, pixels\n}\n\nprivate enum ClientError: LocalizedError {\n\tcase usage, target, image, pixels, dialect\n\n\tvar errorDescription: String? {\n\t\tswitch self {\n\t\tcase .usage:\n\t\t\t\"Usage: desktop-tools-client apps | windows | inspect

[accessibility|pixels] | menus|menu|action|management|launch|open|clipboard < JSON\"\n\t\tcase .target:\n\t\t\t\"The native observation did not confirm the requested process and window. Refresh the window list and try again.\"\n\t\tcase .image:\n\t\t\t\"The native observation returned an empty or oversized screenshot.\"\n\t\tcase .pixels:\n\t\t\t\"The native pixel observation did not publish a screenshot-only snapshot with exact-window coordinate authority.\"\n\t\tcase .dialect:\n\t\t\t\"The desktop host does not serve the \\(desktopToolsProtocol) operations. No request was sent; use the client built with that host\"\n\t\t}\n\t}\n}\n","editable",false,"branch","documentNavigation","repo","githubnext/desktop-tools","repoId","171aa4a4-9481-4286-8421-54a56a68269e","ownerId","org:89615882","site","nexthub","private","admin","settings","maintainer","triage","starred","stars",0,"watchers","watching","pinned","notify","views",[119,120,121,122,123,124,125,126,127,128,129,130,131],"navigation",{"_54":55,"_78":79,"_80":81,"_82":83,"_84":85,"_86":87,"_88":89},"sources",[],"viewer","meta",{"_70":71,"_72":73,"_74":75,"_42":31,"_76":69,"_77":31,"_48":49,"_50":49},"branches",[69],"tags",[],"cloneUrl","https://git.nexthub.sh/githubnext/desktop-tools.git","main","description","Computer use-like tools for macOS. Allows agents to inspect and interface with desktop applications.","owner","githubnext","section","","head","empty","pins",[101,102,103],"documents",{"_98":99,"_100":49},"code",{"_95":96,"_97":49},"automations",{"_92":93,"_94":-5},"delivery",{"_90":49,"_91":-5},"tag","DeGuQgMovPxpL5irDgLqhNcP8A_","releases","applications","workflows",1,"deployments","commits",6,"changes","counts",{},"total",{"_104":97,"_106":117,"_15":97,"_108":118,"_52":110,"_111":31,"_112":110,"_113":49},{"_104":114,"_106":115,"_15":114,"_108":116,"_52":110,"_111":31,"_112":110,"_113":49},{"_104":105,"_106":107,"_15":105,"_108":109,"_52":110,"_111":31,"_112":110,"_113":49},"id","issues","name","Issues","filters",{},true,"saved","pinnable","count","tasks","Tasks",{},"Changes",{},{"_104":97,"_106":117,"_15":97,"_108":162,"_52":110,"_111":31,"_112":110},{"_104":114,"_106":115,"_15":114,"_108":161,"_52":110,"_111":31,"_112":110},{"_104":105,"_106":107,"_15":105,"_108":160,"_52":110,"_111":31,"_112":110},{"_104":157,"_106":158,"_15":157,"_108":159,"_52":31,"_111":31,"_112":110},{"_104":154,"_106":155,"_15":154,"_108":156,"_52":31,"_111":31,"_112":110},{"_104":92,"_106":152,"_15":84,"_108":153,"_52":31,"_111":31,"_112":110},{"_104":94,"_106":149,"_15":150,"_108":151,"_52":31,"_111":31,"_112":110},{"_104":90,"_106":147,"_15":90,"_108":148,"_52":31,"_111":31,"_112":110},{"_104":143,"_106":144,"_15":145,"_108":146,"_52":31,"_111":31,"_112":110},{"_104":80,"_106":141,"_15":80,"_108":142,"_52":31,"_111":31,"_112":31},{"_104":138,"_106":139,"_15":138,"_108":140,"_52":31,"_111":31,"_112":31},{"_104":135,"_106":136,"_15":135,"_108":137,"_52":31,"_111":31,"_112":31},{"_104":132,"_106":133,"_15":132,"_108":134,"_52":31,"_111":31,"_112":31},"decisions","Decisions",{},"changelog","Changelog",{},"roadmap","Roadmap",{},"Documents",{},"activity","Activity","insights",{},"Releases",{},"Deployments","automations/deployments",{},"Workflows",{},"discussions","Discussions",{},"goals","Goals",{},{},{},{},"groups","user","agents",[],"now",1794435314858,"sidebar","expanded"]