Adding measurement cancellation reason diagnostics (#77)

Adding a way to diagnose TTVC measurement cancellations by providing an optional callback that will be called when measurement is cancelled
dropbox · Sep 13, 2023 · 782b7fb · 782b7fb
1 parent 750edb3
commit 782b7fb
Show file tree

Hide file tree

Showing 11 changed files with 312 additions and 46 deletions.
diff --git a/.gitignore b/.gitignore
@@ -109,6 +109,9 @@ lib
 # VS Code settings
 .vscode
 
+# IntelliJ Settings
+.idea
+
 # Playwright
 test-results/
 playwright-report/
diff --git a/README.md b/README.md
@@ -14,6 +14,7 @@
   - [Report metrics to a collection endpoint](#report-metrics-to-a-collection-endpoint)
   - [Record a PerformanceTimeline entry](#record-a-performancetimeline-entry)
   - [Client-side navigation with React Router](#client-side-navigation-with-react-router)
+  - [Attributing TTVC measurement cancellations](#attributing-ttvc-measurement-cancellations)
 - [API](#api)
   - [Types](#types)
   - [Functions](#functions)
@@ -164,6 +165,31 @@ const App = () => {
 };
 ```
 
+### Attributing TTVC measurement cancellations
+
+In certain cases, @dropbox/ttvc might discard the measurement before it is captured.
+
+This can happen if a user interacts with or navigates away from the page, or the page is put in the background before it has reached a state ot visual completeness.
+
+This is done to obtain a higher confidence of the measurement's accuracy, as interaction with a page can cause it to change in ways that invalidate the measurement.
+
+However, @dropbox/ttvc provides a way to monitor these cancellations and attribute them to a specific cause. A second callback function provided to `onTTVC` will be called when the measurement is cancelled.
+
+```js
+import {init, onTTVC} from '@dropbox/ttvc';
+
+init();
+
+onTTVC(
+  (measurement) => {
+    console.log('TTVC measurement captured:', measurement.duration);
+  },
+  (error) => {
+    console.log('TTVC measurement cancelled:', error.cancellationReason);
+  }
+);
+```
+
 ## API
 
 ### Types
@@ -212,6 +238,46 @@ export type Metric = {
 };
 ```
 
+#### `CancellationError`
+
+```typescript
+export type CancellationError = {
+  // time since timeOrigin that the navigation was triggered
+  start: number;
+
+  // time since timeOrigin that cancellation occurred
+  end: number;
+
+  // reason for cancellation
+  cancellationReason: CancellationReason;
+
+  // Optional type of event that triggered cancellation
+  eventType?: string;
+
+  // Optional target of event that triggered cancellation
+  eventTarget?: EventTarget;
+
+  // the most recent visual update; this can be either a mutation or a load event target
+  lastVisibleChange?: HTMLElement | TimestampedMutationRecord;
+
+  navigationType: NavigationType;
+};
+
+export enum CancellationReason {
+  // navigation has occurred
+  NEW_NAVIGATION = 'NEW_NAVIGATION',
+
+  // page was put in background
+  VISIBILITY_CHANGE = 'VISIBILITY_CHANGE',
+
+  // user interaction occurred
+  USER_INTERACTION = 'USER_INTERACTION',
+
+  // manual cancellation via API happened
+  MANUAL_CANCELLATION = 'MANUAL_CANCELLATION',
+}
+```
+
 #### `TtvcOptions`
 
 ```typescript

diff --git a/src/index.ts b/src/index.ts
@@ -3,12 +3,18 @@ import {TtvcOptions, setConfig} from './util/constants';
 import {Logger} from './util/logger';
 import {
   getVisuallyCompleteCalculator,
-  MetricSubscriber,
+  MetricSuccessSubscriber,
+  MetricErrorSubscriber,
   VisuallyCompleteCalculator,
 } from './visuallyCompleteCalculator.js';
 
 export type {TtvcOptions};
-export type {Metric, MetricSubscriber} from './visuallyCompleteCalculator';
+export type {
+  Metric,
+  CancellationError,
+  MetricSuccessSubscriber,
+  MetricErrorSubscriber,
+} from './visuallyCompleteCalculator';
 
 let calculator: VisuallyCompleteCalculator;
 
@@ -53,10 +59,14 @@ export const init = (options?: TtvcOptions) => {
  * @example
  * const unsubscribe = onTTVC(ms => console.log(ms));
  *
- * @param callback Triggered once for each navigation instance.
+ * @param successCallback Triggered once for each navigation instance when TTVC was successfully captured.
+ * @param [errorCallback] Triggered when TTVC failed to capture
  * @returns A function that unsubscribes the callback from this metric.
  */
-export const onTTVC = (callback: MetricSubscriber) => calculator?.onTTVC(callback);
+export const onTTVC = (
+  successCallback: MetricSuccessSubscriber,
+  errorCallback?: MetricErrorSubscriber
+) => calculator?.onTTVC(successCallback, errorCallback);
 
 /**
  * Begin measuring a new navigation.

diff --git a/src/visuallyCompleteCalculator.ts b/src/visuallyCompleteCalculator.ts
@@ -10,6 +10,10 @@ export type NavigationType =
   // Navigation was triggered with a script operation, e.g. in a single page application.
   | 'script';
 
+/**
+ * TTVC `Metric` type uses `PerformanceMeasure` type as it's guideline
+ * https://developer.mozilla.org/en-US/docs/Web/API/PerformanceMeasure
+ */
 export type Metric = {
   // time since timeOrigin that the navigation was triggered
   // (this will be 0 for the initial pageload)
@@ -33,26 +37,73 @@ export type Metric = {
   };
 };
 
-export type MetricSubscriber = (measurement: Metric) => void;
+/**
+ * At the moment, the only error that can occur is measurement cancellation
+ */
+export type CancellationError = {
+  // time since timeOrigin that the navigation was triggered
+  start: number;
+
+  // time since timeOrigin that cancellation occurred
+  end: number;
+
+  // reason for cancellation
+  cancellationReason: CancellationReason;
+
+  // Optional type of event that triggered cancellation
+  eventType?: string;
+
+  // Optional target of event that triggered cancellation
+  eventTarget?: EventTarget;
+
+  // the most recent visual update; this can be either a mutation or a load event target
+  lastVisibleChange?: HTMLElement | TimestampedMutationRecord;
+
+  navigationType: NavigationType;
+};
+
+export const enum CancellationReason {
+  // navigation has occurred
+  NEW_NAVIGATION = 'NEW_NAVIGATION',
+
+  // page was put in background
+  VISIBILITY_CHANGE = 'VISIBILITY_CHANGE',
+
+  // user interaction occurred
+  USER_INTERACTION = 'USER_INTERACTION',
+
+  // manual cancellation via API happened
+  MANUAL_CANCELLATION = 'MANUAL_CANCELLATION',
+}
+
+export type MetricSuccessSubscriber = (measurement: Metric) => void;
+export type MetricErrorSubscriber = (error: CancellationError) => void;
 
 /**
- * TODO: Document
+ * Core of the TTVC calculation that ties viewport observers and network monitoring
+ * into a singleton that facilitates communication of TTVC metric measurement and error
+ * information to subscribers.
  */
 class VisuallyCompleteCalculator {
+  // configuration
   public debug = false;
   public idleTimeout = 200;
 
+  // observers
   private inViewportMutationObserver: InViewportMutationObserver;
   private inViewportImageObserver: InViewportImageObserver;
 
   // measurement state
   private lastMutation?: TimestampedMutationRecord;
   private lastImageLoadTimestamp = -1;
   private lastImageLoadTarget?: HTMLElement;
-  private subscribers = new Set<MetricSubscriber>();
   private navigationCount = 0;
   private activeMeasurementIndex?: number; // only one measurement should be active at a time
 
+  // subscribers
+  private successSubscribers = new Set<MetricSuccessSubscriber>();
+  private errorSubscribers = new Set<MetricErrorSubscriber>();
+
   /**
    * Determine whether the calculator should run in the current environment
    */
@@ -84,15 +135,27 @@ class VisuallyCompleteCalculator {
     });
   }
 
-  /** abort the current TTVC measurement */
-  cancel() {
+  /**
+   * expose a method to abort the current TTVC measurement
+   * @param eventType - type of event that triggered cancellation (note that cancellationReason will be set to "manual" regardless of this value).
+   */
+  cancel(eventType?: string) {
     Logger.info(
       'VisuallyCompleteCalculator.cancel()',
       '::',
       'index =',
       this.activeMeasurementIndex
     );
     this.activeMeasurementIndex = undefined;
+
+    this.error({
+      start: getActivationStart(),
+      end: performance.now(),
+      cancellationReason: CancellationReason.MANUAL_CANCELLATION,
+      eventType: eventType,
+      navigationType: getNavigationType(),
+      lastVisibleChange: this.getLastVisibleChange(),
+    });
   }
 
   /** begin measuring a new navigation */
@@ -114,20 +177,34 @@ class VisuallyCompleteCalculator {
     }
 
     // setup
-    const cancel = () => {
+    const cancel = (e: Event, cancellationReason: CancellationReason) => {
       if (this.activeMeasurementIndex === navigationIndex) {
         this.activeMeasurementIndex = undefined;
+
+        this.error({
+          start,
+          end: performance.now(),
+          cancellationReason,
+          eventType: e.type,
+          eventTarget: e.target || undefined,
+          navigationType,
+          lastVisibleChange: this.getLastVisibleChange(),
+        });
       }
     };
 
+    const cancelOnInteraction = (e: Event) => cancel(e, CancellationReason.USER_INTERACTION);
+    const cancelOnNavigation = (e: Event) => cancel(e, CancellationReason.NEW_NAVIGATION);
+    const cancelOnVisibilityChange = (e: Event) => cancel(e, CancellationReason.VISIBILITY_CHANGE);
+
     this.inViewportImageObserver.observe();
     this.inViewportMutationObserver.observe(document.documentElement);
-    window.addEventListener('pagehide', cancel);
-    window.addEventListener('visibilitychange', cancel);
+    window.addEventListener('pagehide', cancelOnNavigation);
+    window.addEventListener('visibilitychange', cancelOnVisibilityChange);
     // attach user interaction listeners next tick (we don't want to pick up the SPA navigation click)
     window.setTimeout(() => {
-      window.addEventListener('click', cancel);
-      window.addEventListener('keydown', cancel);
+      window.addEventListener('click', cancelOnInteraction);
+      window.addEventListener('keydown', cancelOnInteraction);
     }, 0);
 
     // wait for page to be definitely DONE
@@ -136,7 +213,7 @@ class VisuallyCompleteCalculator {
     // - wait for simultaneous network and CPU idle
     const didNetworkTimeOut = await new Promise<boolean>(requestAllIdleCallback);
 
-    // if this navigation's measurment hasn't been cancelled, record it.
+    // if this navigation's measurement hasn't been cancelled, record it.
     if (navigationIndex === this.activeMeasurementIndex) {
       // identify timestamp of last visible change
       const end = Math.max(start, this.lastImageLoadTimestamp, this.lastMutation?.timestamp ?? 0);
@@ -149,10 +226,7 @@ class VisuallyCompleteCalculator {
         detail: {
           navigationType,
           didNetworkTimeOut,
-          lastVisibleChange:
-            this.lastImageLoadTimestamp > (this.lastMutation?.timestamp ?? 0)
-              ? this.lastImageLoadTarget
-              : this.lastMutation,
+          lastVisibleChange: this.getLastVisibleChange(),
         },
       });
     } else {
@@ -162,13 +236,23 @@ class VisuallyCompleteCalculator {
         'index =',
         navigationIndex
       );
+
+      if (this.activeMeasurementIndex) {
+        this.error({
+          start,
+          end: performance.now(),
+          cancellationReason: CancellationReason.NEW_NAVIGATION,
+          navigationType,
+          lastVisibleChange: this.getLastVisibleChange(),
+        });
+      }
     }
 
     // cleanup
-    window.removeEventListener('pagehide', cancel);
-    window.removeEventListener('visibilitychange', cancel);
-    window.removeEventListener('click', cancel);
-    window.removeEventListener('keydown', cancel);
+    window.removeEventListener('pagehide', cancelOnNavigation);
+    window.removeEventListener('visibilitychange', cancelOnVisibilityChange);
+    window.removeEventListener('click', cancelOnInteraction);
+    window.removeEventListener('keydown', cancelOnInteraction);
     // only disconnect observers if this is the most recent navigation
     if (navigationIndex === this.navigationCount) {
       this.inViewportImageObserver.disconnect();
@@ -200,16 +284,51 @@ class VisuallyCompleteCalculator {
       this.activeMeasurementIndex
     );
     Logger.info('TTVC:', measurement, '::', 'index =', this.activeMeasurementIndex);
-    this.subscribers.forEach((subscriber) => subscriber(measurement));
+    this.successSubscribers.forEach((subscriber) => subscriber(measurement));
+  }
+
+  private error(error: CancellationError) {
+    Logger.debug(
+      'VisuallyCompleteCalculator.error()',
+      '::',
+      'cancellationReason =',
+      error.cancellationReason,
+      '::',
+      'eventType =',
+      error.eventType || 'none',
+      '::',
+      'index =',
+      this.activeMeasurementIndex
+    );
+    this.errorSubscribers.forEach((subscriber) => subscriber(error));
+  }
+
+  private getLastVisibleChange() {
+    return this.lastImageLoadTimestamp > (this.lastMutation?.timestamp ?? 0)
+      ? this.lastImageLoadTarget
+      : this.lastMutation;
   }
 
   /** subscribe to Visually Complete metrics */
-  onTTVC = (subscriber: MetricSubscriber) => {
-    // register subscriber callback
-    this.subscribers.add(subscriber);
+  onTTVC = (
+    successSubscriber: MetricSuccessSubscriber,
+    errorSubscriber?: MetricErrorSubscriber
+  ) => {
+    // register subscriber callbacks
+    this.successSubscribers.add(successSubscriber);
+
+    if (errorSubscriber) {
+      this.errorSubscribers.add(errorSubscriber);
+    }
 
     // return an unsubscribe function
-    return () => this.subscribers.delete(subscriber);
+    return () => {
+      this.successSubscribers.delete(successSubscriber);
+
+      if (errorSubscriber) {
+        this.errorSubscribers.delete(errorSubscriber);
+      }
+    };
   };
 }