From f46e3f3ca99b0a1ad9f799b720ac8505dcaf451f Mon Sep 17 00:00:00 2001 From: CanadaHonk <19228318+CanadaHonk@users.noreply.github.com> Date: Fri, 16 Dec 2022 23:20:03 +0000 Subject: [PATCH] Total internal rewrite (#7) * cdp: initial add * cdp: fix not retrying for port, and firefox * major internal rewrite * roadmap: update for rewrite * package: use correct deps for rewrite * chore: organise into dirs * chore: fix imports * chore: change versioning to 0.x.y * firefox: fix error when window size is undefined --- package.json | 4 +- roadmap.md | 2 + src/browser/chromium.js | 133 +----------------------------------- src/browser/firefox.js | 88 ++---------------------- src/index.js | 2 +- src/launcher/inject.js | 68 ++++++++++++++++++ src/launcher/start.js | 37 ++++++++++ src/lib/cdp.js | 126 ++++++++++++++++++++++++++++++++++ src/{browser => lib}/ipc.js | 0 9 files changed, 245 insertions(+), 215 deletions(-) create mode 100644 src/launcher/inject.js create mode 100644 src/launcher/start.js create mode 100644 src/lib/cdp.js rename src/{browser => lib}/ipc.js (100%) diff --git a/package.json b/package.json index 5e44f44..0f483ca 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gluon-framework/gluon", - "version": "5.1.0", + "version": "0.6.0", "description": "Make websites into desktop apps with system installed browsers and NodeJS.", "main": "src/index.js", "scripts": {}, @@ -16,6 +16,6 @@ "homepage": "https://github.com/gluon-framework/gluon#readme", "type": "module", "dependencies": { - "chrome-remote-interface": "^0.31.3" + "ws": "^8.11.0" } } \ No newline at end of file diff --git a/roadmap.md b/roadmap.md index cf8022b..4ca4c85 100644 --- a/roadmap.md +++ b/roadmap.md @@ -4,7 +4,9 @@ > Want more info on what some of these mean/are? Ask in [our Discord](https://discord.gg/RFtUCA8fST)! ## December 2022 - January 2023 +- [X] Total internal rewrite - [ ] "Hibernation" feasibility study +- [ ] Automated PR/commit CI testing - [ ] System Tray API - [ ] Electron "compatibility layer" for basic/simple apps as a demo of versatile API / pros? - More? Need to know what's wanted \ No newline at end of file diff --git a/src/browser/chromium.js b/src/browser/chromium.js index bd13040..ca51816 100644 --- a/src/browser/chromium.js +++ b/src/browser/chromium.js @@ -1,6 +1,4 @@ -import { spawn } from 'child_process'; - -import makeIPCApi from './ipc.js'; +import StartBrowser from '../launcher/start.js'; const presets = { // Presets from OpenAsar 'base': '--autoplay-policy=no-user-gesture-required --disable-features=WinRetrieveSuggestionsOnlyOnDemand,HardwareMediaKeyHandling,MediaSessionService', // Base Discord @@ -10,136 +8,11 @@ const presets = { // Presets from OpenAsar }; export default async ({ browserName, browserPath, dataPath }, { url, windowSize }) => { - const proc = spawn(browserPath, [ + return await StartBrowser(browserPath, [ `--app=${url}`, `--remote-debugging-pipe`, `--user-data-dir=${dataPath}`, windowSize ? `--window-size=${windowSize.join(',')}` : '', ...`--new-window --disable-extensions --disable-default-apps --disable-breakpad --disable-crashpad --disable-background-networking --disable-domain-reliability --disable-component-update --disable-sync --disable-features=AutofillServerCommunication ${presets.perf}`.split(' ') - ].filter(x => x), { - detached: false, - stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe'] - }); - - proc.stdout.pipe(proc.stdout); - proc.stderr.pipe(proc.stderr); - - // todo: move this to it's own library - const { 3: pipeWrite, 4: pipeRead } = proc.stdio; - - log('connected to CDP over stdio pipe'); - - let onReply = {}, pageLoadCallback = () => {}, onWindowMessage = () => {}; - const onMessage = msg => { - msg = JSON.parse(msg); - - // log('received', msg); - if (onReply[msg.id]) { - onReply[msg.id](msg); - delete onReply[msg.id]; - } - - if (msg.method === 'Runtime.bindingCalled' && msg.name === 'gluonSend') onWindowMessage(JSON.parse(msg.payload)); - if (msg.method === 'Page.frameStoppedLoading') pageLoadCallback(msg.params); - if (msg.method === 'Runtime.executionContextCreated') injectIPC(); // ensure IPC injection again - }; - - let msgId = 0; - const sendMessage = async (method, params = {}, sessionId = undefined) => { - const id = msgId++; - - const msg = { - id, - method, - params - }; - - if (sessionId) msg.sessionId = sessionId; - - pipeWrite.write(JSON.stringify(msg)); - pipeWrite.write('\0'); - - // log('sent', msg); - - const reply = await new Promise(res => { - onReply[id] = msg => res(msg); - }); - - return reply.result; - }; - - let pending = ''; - pipeRead.on('data', buf => { - let end = buf.indexOf('\0'); // messages are null separated - - if (end === -1) { // no complete message yet - pending += buf.toString(); - return; - } - - let start = 0; - while (end !== -1) { // while we have pending complete messages, dispatch them - const message = pending + buf.toString(undefined, start, end); // get next whole message - onMessage(message); - - start = end + 1; // find next ending - end = buf.indexOf('\0', start); - pending = ''; - } - - pending = buf.toString(undefined, start); // update pending with current pending - }); - - pipeRead.on('close', () => log('pipe read closed')); - - // await new Promise(res => setTimeout(res, 1000)); - - let browserInfo; - sendMessage('Browser.getVersion').then(x => { // get browser info async as not important - browserInfo = x; - log('browser:', x.product); - }); - - const target = (await sendMessage('Target.getTargets')).targetInfos[0]; - - const { sessionId } = await sendMessage('Target.attachToTarget', { - targetId: target.targetId, - flatten: true - }); - - sendMessage('Runtime.enable', {}, sessionId); // enable runtime API - - sendMessage('Runtime.addBinding', { // setup sending from window to Node via Binding - name: '_gluonSend' - }, sessionId); - - const evalInWindow = async func => { - return await sendMessage(`Runtime.evaluate`, { - expression: typeof func === 'string' ? func : `(${func.toString()})()` - }, sessionId); - }; - - const [ ipcMessageCallback, injectIPC, IPCApi ] = await makeIPCApi({ - browserName, - browserInfo - }, { - evaluate: params => sendMessage(`Runtime.evaluate`, params, sessionId), - addScriptToEvaluateOnNewDocument: params => sendMessage('Page.addScriptToEvaluateOnNewDocument', params, sessionId), - pageLoadPromise: new Promise(res => pageLoadCallback = res) - }); - onWindowMessage = ipcMessageCallback; - - log('finished setup'); - - return { - window: { - eval: evalInWindow, - }, - - ipc: IPCApi, - - cdp: { - send: (method, params) => sendMessage(method, params, sessionId) - } - }; + ], 'stdio', { browserName }); }; \ No newline at end of file diff --git a/src/browser/firefox.js b/src/browser/firefox.js index a76f031..a361930 100644 --- a/src/browser/firefox.js +++ b/src/browser/firefox.js @@ -1,21 +1,10 @@ import { mkdir, writeFile } from 'fs/promises'; import { join } from 'path'; -import { spawn } from 'child_process'; -let CDP; -try { - CDP = (await import('chrome-remote-interface')).default; -} catch { - console.warn('Dependencies for Firefox are not installed!'); -} +import StartBrowser from '../launcher/start.js'; -import makeIPCApi from './ipc.js'; - -const portRange = [ 10000, 60000 ]; export default async ({ browserName, browserPath, dataPath }, { url, windowSize }) => { - const debugPort = Math.floor(Math.random() * (portRange[1] - portRange[0] + 1)) + portRange[0]; - await mkdir(dataPath, { recursive: true }); await writeFile(join(dataPath, 'user.js'), ` user_pref("toolkit.legacyUserProfileCustomizations.stylesheets", true); @@ -24,8 +13,8 @@ user_pref('devtools.debugger.prompt-connection', false); user_pref('devtools.debugger.remote-enabled', true); user_pref('toolkit.telemetry.reportingpolicy.firstRun', false); user_pref('browser.shell.checkDefaultBrowser', false); -user_pref('privacy.window.maxInnerWidth', ${windowSize[0]}); -user_pref('privacy.window.maxInnerHeight', ${windowSize[1]}); +${!windowSize ? '' : `user_pref('privacy.window.maxInnerWidth', ${windowSize[0]}); +user_pref('privacy.window.maxInnerHeight', ${windowSize[1]});`} user_pref('privacy.resistFingerprinting', true); user_pref('fission.bfcacheInParent', false); user_pref('fission.webContentIsolationStrategy', 0); @@ -82,76 +71,11 @@ html:not([tabsintitlebar="true"]) .tab-icon-image { } `); - const proc = spawn(browserPath, [ - `--remote-debugging-port=${debugPort}`, - `-window-size`, windowSize.join(','), + return await StartBrowser(browserPath, [ + ...(!windowSize ? [] : [ `-window-size`, windowSize.join(',') ]), `-profile`, dataPath, `-new-window`, url, `-new-instance`, `-no-remote` - ].filter(x => x), { - detached: false, - stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe'] - }); - - proc.stdout.pipe(proc.stdout); - proc.stderr.pipe(proc.stderr); - - log(`connecting to CDP over websocket (${debugPort})...`); - - let CDPInstance; - const connect = async () => { - try { - CDPInstance = await CDP({ - port: debugPort - }); - } catch { - await new Promise(res => setTimeout(res)); - await connect(); - } - }; - - await connect(); - - log(`connected to CDP over websocket (${debugPort})`); - - const { Browser, Runtime, Page } = CDPInstance; - - const browserInfo = await Browser.getVersion(); - log('browser:', browserInfo.product); - - await Runtime.enable(); - - /* Runtime.addBinding({ - name: '_gluonSend' - }); */ - - const [ ipcMessageCallback, injectIPC, IPCApi ] = await makeIPCApi({ - browserName, - browserInfo - }, { - evaluate: Runtime.evaluate, - addScriptToEvaluateOnNewDocument: Page.addScriptToEvaluateOnNewDocument, - pageLoadPromise: new Promise(res => Page.frameStoppedLoading(res)) - }); - - // todo: IPC Node -> Web for Firefox - - log('finished setup'); - - return { - window: { - eval: async func => { - return await Runtime.evaluate({ - expression: typeof func === 'string' ? func : `(${func.toString()})()` - }); - } - }, - - ipc: IPCApi, - - cdp: { // todo: public CDP API for Firefox - send: () => {} - } - }; + ], 'websocket', { browserName }); }; \ No newline at end of file diff --git a/src/index.js b/src/index.js index f1a4804..88d6f73 100644 --- a/src/index.js +++ b/src/index.js @@ -1,7 +1,7 @@ const rgb = (r, g, b, msg) => `\x1b[38;2;${r};${g};${b}m${msg}\x1b[0m`; global.log = (...args) => console.log(`[${rgb(88, 101, 242, 'Gluon')}]`, ...args); -process.versions.gluon = '5.1-dev'; +process.versions.gluon = '0.6.0'; import { join, dirname, delimiter, sep } from 'path'; import { access, readdir } from 'fs/promises'; diff --git a/src/launcher/inject.js b/src/launcher/inject.js new file mode 100644 index 0000000..e6b4bed --- /dev/null +++ b/src/launcher/inject.js @@ -0,0 +1,68 @@ +import IPCApi from '../lib/ipc.js'; + +export default async (CDP, injectionType = 'browser', { browserName }) => { + let pageLoadCallback = () => {}, onWindowMessage = () => {}; + CDP.onMessage(msg => { + if (msg.method === 'Runtime.bindingCalled' && msg.name === 'gluonSend') onWindowMessage(JSON.parse(msg.payload)); + if (msg.method === 'Page.frameStoppedLoading') pageLoadCallback(msg.params); + if (msg.method === 'Runtime.executionContextCreated') injectIPC(); // ensure IPC injection again + }); + + + let browserInfo, sessionId; + if (injectionType === 'browser') { // connected to browser itself, need to get and attach to a target + CDP.sendMessage('Browser.getVersion').then(x => { // get browser info async as we have time while attaching + browserInfo = x; + log('browser:', x.product); + }); + + const target = (await CDP.sendMessage('Target.getTargets')).targetInfos[0]; + + sessionId = (await CDP.sendMessage('Target.attachToTarget', { + targetId: target.targetId, + flatten: true + })).sessionId; + } else { // already attached to target + browserInfo = await CDP.sendMessage('Browser.getVersion'); + log('browser:', browserInfo.product); + } + + + CDP.sendMessage('Runtime.enable', {}, sessionId); // enable runtime API + + CDP.sendMessage('Runtime.addBinding', { // setup sending from window to Node via Binding + name: '_gluonSend' + }, sessionId); + + const evalInWindow = async func => { + return await CDP.sendMessage(`Runtime.evaluate`, { + expression: typeof func === 'string' ? func : `(${func.toString()})()` + }, sessionId); + }; + + + const [ ipcMessageCallback, injectIPC, IPC ] = await IPCApi({ + browserName, + browserInfo + }, { + evaluate: params => CDP.sendMessage(`Runtime.evaluate`, params, sessionId), + addScriptToEvaluateOnNewDocument: params => CDP.sendMessage('Page.addScriptToEvaluateOnNewDocument', params, sessionId), + pageLoadPromise: new Promise(res => pageLoadCallback = res) + }); + onWindowMessage = ipcMessageCallback; + + + log('finished setup'); + + return { + window: { + eval: evalInWindow, + }, + + ipc: IPC, + + cdp: { + send: (method, params) => CDP.sendMessage(method, params, sessionId) + } + }; +}; \ No newline at end of file diff --git a/src/launcher/start.js b/src/launcher/start.js new file mode 100644 index 0000000..2143561 --- /dev/null +++ b/src/launcher/start.js @@ -0,0 +1,37 @@ +import { spawn } from 'child_process'; + +import ConnectCDP from '../lib/cdp.js'; +import InjectInto from './inject.js'; + +const portRange = [ 10000, 60000 ]; + +export default async (browserPath, args, transport, extra) => { + const port = transport === 'websocket' ? (Math.floor(Math.random() * (portRange[1] - portRange[0] + 1)) + portRange[0]) : null; + + const proc = spawn(browserPath, [ + transport === 'stdio' ? `--remote-debugging-pipe` : `--remote-debugging-port=${port}`, + ...args + ].filter(x => x), { + detached: false, + stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe'] + }); + + proc.stdout.pipe(proc.stdout); + proc.stderr.pipe(proc.stderr); + + log(`connecting to CDP over ${transport === 'stdio' ? 'stdio pipe' : `websocket (${port})`}...`); + + let CDP; + switch (transport) { + case 'websocket': + CDP = await ConnectCDP({ port }); + break; + + case 'stdio': + const { 3: pipeWrite, 4: pipeRead } = proc.stdio; + CDP = await ConnectCDP({ pipe: { pipeWrite, pipeRead } }); + break; + } + + return await InjectInto(CDP, transport === 'stdio' ? 'browser' : 'target', extra); +}; \ No newline at end of file diff --git a/src/lib/cdp.js b/src/lib/cdp.js new file mode 100644 index 0000000..0f9ace8 --- /dev/null +++ b/src/lib/cdp.js @@ -0,0 +1,126 @@ +import WebSocket from 'ws'; +import { get } from 'http'; + +export default async ({ pipe: { pipeWrite, pipeRead } = {}, port }) => { + let messageCallbacks = [], onReply = {}; + const onMessage = msg => { + msg = JSON.parse(msg); + + // log('received', msg); + if (onReply[msg.id]) { + onReply[msg.id](msg); + delete onReply[msg.id]; + + return; + } + + for (const callback of messageCallbacks) callback(msg); + }; + + let _send; + + let msgId = 0; + const sendMessage = async (method, params = {}, sessionId = undefined) => { + const id = msgId++; + + const msg = { + id, + method, + params + }; + + if (sessionId) msg.sessionId = sessionId; + + _send(JSON.stringify(msg)); + + // log('sent', msg); + + const reply = await new Promise(res => { + onReply[id] = msg => res(msg); + }); + + return reply.result; + }; + + if (port) { + const continualTrying = func => new Promise(resolve => { + const attempt = async () => { + try { + process.stdout.write('.'); + resolve(await func()); + } catch (e) { // fail, wait 100ms and try again + await new Promise(res => setTimeout(res, 200)); + await attempt(); + } + }; + + attempt(); + }); + + const targets = await continualTrying(() => new Promise((resolve, reject) => get(`http://127.0.0.1:${port}/json/list`, res => { + let body = ''; + res.on('data', chunk => body += chunk.toString()); + res.on('end', () => { + try { + resolve(JSON.parse(body)) + } catch { + reject(); + } + }); + }).on('error', reject))); + + console.log(); + + const target = targets[0]; + + log('got target', target); + + const ws = new WebSocket(target.webSocketDebuggerUrl); + await new Promise(resolve => ws.on('open', resolve)); + + _send = data => ws.send(data); + + ws.on('message', data => onMessage(data)); + } else { + let pending = ''; + pipeRead.on('data', buf => { + let end = buf.indexOf('\0'); // messages are null separated + + if (end === -1) { // no complete message yet + pending += buf.toString(); + return; + } + + let start = 0; + while (end !== -1) { // while we have pending complete messages, dispatch them + const message = pending + buf.toString(undefined, start, end); // get next whole message + onMessage(message); + + start = end + 1; // find next ending + end = buf.indexOf('\0', start); + pending = ''; + } + + pending = buf.toString(undefined, start); // update pending with current pending + }); + + pipeRead.on('close', () => log('pipe read closed')); + + _send = data => { + pipeWrite.write(data); + pipeWrite.write('\0'); + }; + } + + return { + onMessage: (_callback, once = false) => { + const callback = once ? msg => { + _callback(msg); + messageCallbacks.splice(messageCallbacks.indexOf(callback), 1); // remove self + } : _callback; + + messageCallbacks.push(callback); + }, + sendMessage + }; +}; \ No newline at end of file diff --git a/src/browser/ipc.js b/src/lib/ipc.js similarity index 100% rename from src/browser/ipc.js rename to src/lib/ipc.js