Total internal rewrite (#7)

* cdp: initial add

* cdp: fix not retrying for port, and firefox

* major internal rewrite

* roadmap: update for rewrite

* package: use correct deps for rewrite

* chore: organise into dirs

* chore: fix imports

* chore: change versioning to 0.x.y

* firefox: fix error when window size is undefined

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gluon-framework/gluon",
-  "version": "5.1.0",
+  "version": "0.6.0",
   "description": "Make websites into desktop apps with system installed browsers and NodeJS.",
   "main": "src/index.js",
   "scripts": {},
@@ -16,6 +16,6 @@
   "homepage": "https://github.com/gluon-framework/gluon#readme",
   "type": "module",
   "dependencies": {
-    "chrome-remote-interface": "^0.31.3"
+    "ws": "^8.11.0"
   }
 }

roadmap.md

@@ -4,7 +4,9 @@
 > Want more info on what some of these mean/are? Ask in [our Discord](https://discord.gg/RFtUCA8fST)!
 
 ## December 2022 - January 2023
+- [X] Total internal rewrite
 - [ ] "Hibernation" feasibility study
+- [ ] Automated PR/commit CI testing
 - [ ] System Tray API
 - [ ] Electron "compatibility layer" for basic/simple apps as a demo of versatile API / pros?
 - More? Need to know what's wanted

src/browser/chromium.js

@@ -1,6 +1,4 @@
-import { spawn } from 'child_process';
-
-import makeIPCApi from './ipc.js';
+import StartBrowser from '../launcher/start.js';
 
 const presets = { // Presets from OpenAsar
   'base': '--autoplay-policy=no-user-gesture-required --disable-features=WinRetrieveSuggestionsOnlyOnDemand,HardwareMediaKeyHandling,MediaSessionService', // Base Discord
@@ -10,136 +8,11 @@ const presets = { // Presets from OpenAsar
 };
 
 export default async ({ browserName, browserPath, dataPath }, { url, windowSize }) => {
-  const proc = spawn(browserPath, [
+  return await StartBrowser(browserPath, [
     `--app=${url}`,
     `--remote-debugging-pipe`,
     `--user-data-dir=${dataPath}`,
     windowSize ? `--window-size=${windowSize.join(',')}` : '',
     ...`--new-window --disable-extensions --disable-default-apps --disable-breakpad --disable-crashpad --disable-background-networking --disable-domain-reliability --disable-component-update --disable-sync --disable-features=AutofillServerCommunication ${presets.perf}`.split(' ')
-  ].filter(x => x), {
-    detached: false,
-    stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe']
-  });
-
-  proc.stdout.pipe(proc.stdout);
-  proc.stderr.pipe(proc.stderr);
-
-  // todo: move this to it's own library
-  const { 3: pipeWrite, 4: pipeRead } = proc.stdio;
-
-  log('connected to CDP over stdio pipe');
-
-  let onReply = {}, pageLoadCallback = () => {}, onWindowMessage = () => {};
-  const onMessage = msg => {
-    msg = JSON.parse(msg);
-
-    // log('received', msg);
-    if (onReply[msg.id]) {
-      onReply[msg.id](msg);
-      delete onReply[msg.id];
-    }
-
-    if (msg.method === 'Runtime.bindingCalled' && msg.name === 'gluonSend') onWindowMessage(JSON.parse(msg.payload));
-    if (msg.method === 'Page.frameStoppedLoading') pageLoadCallback(msg.params);
-    if (msg.method === 'Runtime.executionContextCreated') injectIPC(); // ensure IPC injection again
-  };
-
-  let msgId = 0;
-  const sendMessage = async (method, params = {}, sessionId = undefined) => {
-    const id = msgId++;
-
-    const msg = {
-      id,
-      method,
-      params
-    };
-
-    if (sessionId) msg.sessionId = sessionId;
-
-    pipeWrite.write(JSON.stringify(msg));
-    pipeWrite.write('\0');
-
-    // log('sent', msg);
-
-    const reply = await new Promise(res => {
-      onReply[id] = msg => res(msg);
-    });
-
-    return reply.result;
-  };
-
-  let pending = '';
-  pipeRead.on('data', buf => {
-    let end = buf.indexOf('\0'); // messages are null separated
-
-    if (end === -1) { // no complete message yet
-      pending += buf.toString();
-      return;
-    }
-
-    let start = 0;
-    while (end !== -1) { // while we have pending complete messages, dispatch them
-      const message = pending + buf.toString(undefined, start, end); // get next whole message
-      onMessage(message);
-
-      start = end + 1; // find next ending
-      end = buf.indexOf('\0', start);
-      pending = '';
-    }
-
-    pending = buf.toString(undefined, start); // update pending with current pending
-  });
-
-  pipeRead.on('close', () => log('pipe read closed'));
-
-  // await new Promise(res => setTimeout(res, 1000));
-
-  let browserInfo;
-  sendMessage('Browser.getVersion').then(x => { // get browser info async as not important
-    browserInfo = x;
-    log('browser:', x.product);
-  });
-
-  const target = (await sendMessage('Target.getTargets')).targetInfos[0];
-
-  const { sessionId } = await sendMessage('Target.attachToTarget', {
-    targetId: target.targetId,
-    flatten: true
-  });
-
-  sendMessage('Runtime.enable', {}, sessionId); // enable runtime API
-
-  sendMessage('Runtime.addBinding', { // setup sending from window to Node via Binding
-    name: '_gluonSend'
-  }, sessionId);
-
-  const evalInWindow = async func => {
-    return await sendMessage(`Runtime.evaluate`, {
-      expression: typeof func === 'string' ? func : `(${func.toString()})()`
-    }, sessionId);
-  };
-
-  const [ ipcMessageCallback, injectIPC, IPCApi ] = await makeIPCApi({
-    browserName,
-    browserInfo
-  }, {
-    evaluate: params => sendMessage(`Runtime.evaluate`, params, sessionId),
-    addScriptToEvaluateOnNewDocument: params => sendMessage('Page.addScriptToEvaluateOnNewDocument', params, sessionId),
-    pageLoadPromise: new Promise(res => pageLoadCallback = res)
-  });
-  onWindowMessage = ipcMessageCallback;
-
-  log('finished setup');
-
-  return {
-    window: {
-      eval: evalInWindow,
-    },
-
-    ipc: IPCApi,
-
-    cdp: {
-      send: (method, params) => sendMessage(method, params, sessionId)
-    }
-  };
+  ], 'stdio', { browserName });
 };

src/browser/firefox.js

@@ -1,21 +1,10 @@
 import { mkdir, writeFile } from 'fs/promises';
 import { join } from 'path';
-import { spawn } from 'child_process';
 
-let CDP;
-try {
-  CDP = (await import('chrome-remote-interface')).default;
-} catch {
-  console.warn('Dependencies for Firefox are not installed!');
-}
+import StartBrowser from '../launcher/start.js';
 
-import makeIPCApi from './ipc.js';
-
-const portRange = [ 10000, 60000 ];
 
 export default async ({ browserName, browserPath, dataPath }, { url, windowSize }) => {
-  const debugPort = Math.floor(Math.random() * (portRange[1] - portRange[0] + 1)) + portRange[0];
-
   await mkdir(dataPath, { recursive: true });
   await writeFile(join(dataPath, 'user.js'), `
 user_pref("toolkit.legacyUserProfileCustomizations.stylesheets", true);
@@ -24,8 +13,8 @@ user_pref('devtools.debugger.prompt-connection', false);
 user_pref('devtools.debugger.remote-enabled', true);
 user_pref('toolkit.telemetry.reportingpolicy.firstRun', false);
 user_pref('browser.shell.checkDefaultBrowser', false);
-user_pref('privacy.window.maxInnerWidth', ${windowSize[0]});
-user_pref('privacy.window.maxInnerHeight', ${windowSize[1]});
+${!windowSize ? '' : `user_pref('privacy.window.maxInnerWidth', ${windowSize[0]});
+user_pref('privacy.window.maxInnerHeight', ${windowSize[1]});`}
 user_pref('privacy.resistFingerprinting', true);
 user_pref('fission.bfcacheInParent', false);
 user_pref('fission.webContentIsolationStrategy', 0);
@@ -82,76 +71,11 @@ html:not([tabsintitlebar="true"]) .tab-icon-image {
 }
 `);
 
-  const proc = spawn(browserPath, [
-    `--remote-debugging-port=${debugPort}`,
-    `-window-size`, windowSize.join(','),
+  return await StartBrowser(browserPath, [
+    ...(!windowSize ? [] : [ `-window-size`, windowSize.join(',') ]),
     `-profile`, dataPath,
     `-new-window`, url,
     `-new-instance`,
     `-no-remote`
-  ].filter(x => x), {
-    detached: false,
-    stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe']
-  });
-
-  proc.stdout.pipe(proc.stdout);
-  proc.stderr.pipe(proc.stderr);
-
-  log(`connecting to CDP over websocket (${debugPort})...`);
-
-  let CDPInstance;
-  const connect = async () => {
-    try {
-      CDPInstance = await CDP({
-        port: debugPort
-      });
-    } catch {
-      await new Promise(res => setTimeout(res));
-      await connect();
-    }
-  };
-
-  await connect();
-
-  log(`connected to CDP over websocket (${debugPort})`);
-
-  const { Browser, Runtime, Page } = CDPInstance;
-
-  const browserInfo = await Browser.getVersion();
-  log('browser:', browserInfo.product);
-
-  await Runtime.enable();
-
-  /* Runtime.addBinding({
-    name: '_gluonSend'
-  }); */
-
-  const [ ipcMessageCallback, injectIPC, IPCApi ] = await makeIPCApi({
-    browserName,
-    browserInfo
-  }, {
-    evaluate: Runtime.evaluate,
-    addScriptToEvaluateOnNewDocument: Page.addScriptToEvaluateOnNewDocument,
-    pageLoadPromise: new Promise(res => Page.frameStoppedLoading(res))
-  });
-
-  // todo: IPC Node -> Web for Firefox
-
-  log('finished setup');
-
-  return {
-    window: {
-      eval:  async func => {
-        return await Runtime.evaluate({
-          expression: typeof func === 'string' ? func : `(${func.toString()})()`
-        });
-      }
-    },
-
-    ipc: IPCApi,
-
-    cdp: { // todo: public CDP API for Firefox
-      send: () => {}
-    }
-  };
+  ], 'websocket', { browserName });
 };

src/index.js

@@ -1,7 +1,7 @@
 const rgb = (r, g, b, msg) => `\x1b[38;2;${r};${g};${b}m${msg}\x1b[0m`;
 global.log = (...args) => console.log(`[${rgb(88, 101, 242, 'Gluon')}]`, ...args);
 
-process.versions.gluon = '5.1-dev';
+process.versions.gluon = '0.6.0';
 
 import { join, dirname, delimiter, sep } from 'path';
 import { access, readdir } from 'fs/promises';

src/launcher/inject.js

@@ -0,0 +1,68 @@
+import IPCApi from '../lib/ipc.js';
+
+export default async (CDP, injectionType = 'browser', { browserName }) => {
+  let pageLoadCallback = () => {}, onWindowMessage = () => {};
+  CDP.onMessage(msg => {
+    if (msg.method === 'Runtime.bindingCalled' && msg.name === 'gluonSend') onWindowMessage(JSON.parse(msg.payload));
+    if (msg.method === 'Page.frameStoppedLoading') pageLoadCallback(msg.params);
+    if (msg.method === 'Runtime.executionContextCreated') injectIPC(); // ensure IPC injection again
+  });
+
+
+  let browserInfo, sessionId;
+  if (injectionType === 'browser') { // connected to browser itself, need to get and attach to a target
+    CDP.sendMessage('Browser.getVersion').then(x => { // get browser info async as we have time while attaching
+      browserInfo = x;
+      log('browser:', x.product);
+    });
+
+    const target = (await CDP.sendMessage('Target.getTargets')).targetInfos[0];
+
+    sessionId = (await CDP.sendMessage('Target.attachToTarget', {
+      targetId: target.targetId,
+      flatten: true
+    })).sessionId;
+  } else { // already attached to target
+    browserInfo = await CDP.sendMessage('Browser.getVersion');
+    log('browser:', browserInfo.product);
+  }
+
+
+  CDP.sendMessage('Runtime.enable', {}, sessionId); // enable runtime API
+
+  CDP.sendMessage('Runtime.addBinding', { // setup sending from window to Node via Binding
+    name: '_gluonSend'
+  }, sessionId);
+
+  const evalInWindow = async func => {
+    return await CDP.sendMessage(`Runtime.evaluate`, {
+      expression: typeof func === 'string' ? func : `(${func.toString()})()`
+    }, sessionId);
+  };
+
+
+  const [ ipcMessageCallback, injectIPC, IPC ] = await IPCApi({
+    browserName,
+    browserInfo
+  }, {
+    evaluate: params => CDP.sendMessage(`Runtime.evaluate`, params, sessionId),
+    addScriptToEvaluateOnNewDocument: params => CDP.sendMessage('Page.addScriptToEvaluateOnNewDocument', params, sessionId),
+    pageLoadPromise: new Promise(res => pageLoadCallback = res)
+  });
+  onWindowMessage = ipcMessageCallback;
+
+
+  log('finished setup');
+
+  return {
+    window: {
+      eval: evalInWindow,
+    },
+
+    ipc: IPC,
+
+    cdp: {
+      send: (method, params) => CDP.sendMessage(method, params, sessionId)
+    }
+  };
+};

src/launcher/start.js

@@ -0,0 +1,37 @@
+import { spawn } from 'child_process';
+
+import ConnectCDP from '../lib/cdp.js';
+import InjectInto from './inject.js';
+
+const portRange = [ 10000, 60000 ];
+
+export default async (browserPath, args, transport, extra) => {
+  const port = transport === 'websocket' ? (Math.floor(Math.random() * (portRange[1] - portRange[0] + 1)) + portRange[0]) : null;
+
+  const proc = spawn(browserPath, [
+    transport === 'stdio' ? `--remote-debugging-pipe` : `--remote-debugging-port=${port}`,
+    ...args
+  ].filter(x => x), {
+    detached: false,
+    stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe']
+  });
+
+  proc.stdout.pipe(proc.stdout);
+  proc.stderr.pipe(proc.stderr);
+
+  log(`connecting to CDP over ${transport === 'stdio' ? 'stdio pipe' : `websocket (${port})`}...`);
+
+  let CDP;
+  switch (transport) {
+    case 'websocket':
+      CDP = await ConnectCDP({ port });
+      break;
+
+    case 'stdio':
+      const { 3: pipeWrite, 4: pipeRead } = proc.stdio;
+      CDP = await ConnectCDP({ pipe: { pipeWrite, pipeRead } });
+      break;
+  }
+
+  return await InjectInto(CDP, transport === 'stdio' ? 'browser' : 'target', extra);
+};

src/lib/cdp.js

@@ -0,0 +1,126 @@
+import WebSocket from 'ws';
+import { get } from 'http';
+
+export default async ({ pipe: { pipeWrite, pipeRead } = {}, port }) => {
+  let messageCallbacks = [], onReply = {};
+  const onMessage = msg => {
+    msg = JSON.parse(msg);
+
+    // log('received', msg);
+    if (onReply[msg.id]) {
+      onReply[msg.id](msg);
+      delete onReply[msg.id];
+
+      return;
+    }
+
+    for (const callback of messageCallbacks) callback(msg);
+  };
+
+  let _send;
+
+  let msgId = 0;
+  const sendMessage = async (method, params = {}, sessionId = undefined) => {
+    const id = msgId++;
+
+    const msg = {
+      id,
+      method,
+      params
+    };
+
+    if (sessionId) msg.sessionId = sessionId;
+
+    _send(JSON.stringify(msg));
+
+    // log('sent', msg);
+
+    const reply = await new Promise(res => {
+      onReply[id] = msg => res(msg);
+    });
+
+    return reply.result;
+  };
+
+  if (port) {
+    const continualTrying = func => new Promise(resolve => {
+      const attempt = async () => {
+        try {
+          process.stdout.write('.');
+          resolve(await func());
+        } catch (e) { // fail, wait 100ms and try again
+          await new Promise(res => setTimeout(res, 200));
+          await attempt();
+        }
+      };
+
+      attempt();
+    });
+
+    const targets = await continualTrying(() => new Promise((resolve, reject) => get(`http://127.0.0.1:${port}/json/list`, res => {
+      let body = '';
+      res.on('data', chunk => body += chunk.toString());
+      res.on('end', () => {
+        try {
+          resolve(JSON.parse(body))
+        } catch {
+          reject();
+        }
+      });
+    }).on('error', reject)));
+
+    console.log();
+
+    const target = targets[0];
+
+    log('got target', target);
+
+    const ws = new WebSocket(target.webSocketDebuggerUrl);
+    await new Promise(resolve => ws.on('open', resolve));
+
+    _send = data => ws.send(data);
+
+    ws.on('message', data => onMessage(data));
+  } else {
+    let pending = '';
+    pipeRead.on('data', buf => {
+      let end = buf.indexOf('\0'); // messages are null separated
+
+      if (end === -1) { // no complete message yet
+        pending += buf.toString();
+        return;
+      }
+
+      let start = 0;
+      while (end !== -1) { // while we have pending complete messages, dispatch them
+        const message = pending + buf.toString(undefined, start, end); // get next whole message
+        onMessage(message);
+
+        start = end + 1; // find next ending
+        end = buf.indexOf('\0', start);
+        pending = '';
+      }
+
+      pending = buf.toString(undefined, start); // update pending with current pending
+    });
+
+    pipeRead.on('close', () => log('pipe read closed'));
+
+    _send = data => {
+      pipeWrite.write(data);
+      pipeWrite.write('\0');
+    };
+  }
+
+  return {
+    onMessage: (_callback, once = false) => {
+      const callback = once ? msg => {
+        _callback(msg);
+        messageCallbacks.splice(messageCallbacks.indexOf(callback), 1); // remove self
+      } : _callback;
+
+      messageCallbacks.push(callback);
+    },
+    sendMessage
+  };
+};