Skip to content

Commit

Permalink
fix: ensure that we capture service worker requests (cypress-io#28517)
Browse files Browse the repository at this point in the history
* fix: ensure that we capture service worker requests

* add changelog

* fix changelog

* fix tests

* PR comments

* PR comments

* PR comment

* PR comment

* update changelog

* Update cli/CHANGELOG.md

Co-authored-by: Mike McCready <66998419+MikeMcC399@users.noreply.github.com>

* enable builds on all archs

* fix permission issue

* PR comments

* Update smoke.js

* Update cli/CHANGELOG.md

* attempt to fix smoke tests

* bump ci cache

* Update smoke.js

* Update smoke.js

* Update example.json

* fix multiple specs

* fix tests

* Update CHANGELOG.md

---------

Co-authored-by: Mike McCready <66998419+MikeMcC399@users.noreply.github.com>
  • Loading branch information
ryanthemanuel and MikeMcC399 committed Jan 6, 2024
1 parent 7a9e3a4 commit c6f5e9a
Show file tree
Hide file tree
Showing 43 changed files with 1,668 additions and 98 deletions.
13 changes: 4 additions & 9 deletions .circleci/workflows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,7 @@ macWorkflowFilters: &darwin-workflow-filters
# use the following branch as well to ensure that v8 snapshot cache updates are fully tested
- equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
- equal: [ 'feature/experimental-retries', << pipeline.git.branch >> ]
- equal: [ 'chore/update_webpack_deps_to_latest_webpack4_compat', << pipeline.git.branch >> ]
- equal: [ 'lerna-optimize-tasks', << pipeline.git.branch >> ]
- equal: [ 'ryanm/fix/service-worker-capture', << pipeline.git.branch >> ]
- matches:
pattern: /^release\/\d+\.\d+\.\d+$/
value: << pipeline.git.branch >>
Expand All @@ -56,8 +55,7 @@ linuxArm64WorkflowFilters: &linux-arm64-workflow-filters
# use the following branch as well to ensure that v8 snapshot cache updates are fully tested
- equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
- equal: [ 'feature/experimental-retries', << pipeline.git.branch >> ]
- equal: [ 'chore/update_webpack_deps_to_latest_webpack4_compat', << pipeline.git.branch >> ]
- equal: [ 'lerna-optimize-tasks', << pipeline.git.branch >> ]
- equal: [ 'ryanm/fix/service-worker-capture', << pipeline.git.branch >> ]
- matches:
pattern: /^release\/\d+\.\d+\.\d+$/
value: << pipeline.git.branch >>
Expand All @@ -81,10 +79,7 @@ windowsWorkflowFilters: &windows-workflow-filters
# use the following branch as well to ensure that v8 snapshot cache updates are fully tested
- equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
- equal: [ 'feature/experimental-retries', << pipeline.git.branch >> ]
- equal: [ 'chore/update_webpack_deps_to_latest_webpack4_compat', << pipeline.git.branch >> ]
- equal: [ 'lerna-optimize-tasks', << pipeline.git.branch >> ]
- equal: [ 'em/shallow-checkout', << pipeline.git.branch >> ]
- equal: [ 'mschile/mochaEvents_win_sep', << pipeline.git.branch >> ]
- equal: [ 'ryanm/fix/service-worker-capture', << pipeline.git.branch >> ]
- matches:
pattern: /^release\/\d+\.\d+\.\d+$/
value: << pipeline.git.branch >>
Expand Down Expand Up @@ -154,7 +149,7 @@ commands:
name: Set environment variable to determine whether or not to persist artifacts
command: |
echo "Setting SHOULD_PERSIST_ARTIFACTS variable"
echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "publish-binary" && "$CIRCLE_BRANCH" != "em/protocol-log-false" ]]; then
echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "publish-binary" && "$CIRCLE_BRANCH" != "ryanm/fix/service-worker-capture" ]]; then
export SHOULD_PERSIST_ARTIFACTS=true
fi' >> "$BASH_ENV"
# You must run `setup_should_persist_artifacts` command and be using bash before running this command
Expand Down
1 change: 1 addition & 0 deletions cli/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ _Released 1/16/2024 (PENDING)_
- When generating assertions via Cypress Studio, the preview of the generated assertions now correctly displays the past tense of 'expected' instead of 'expect'. Fixed in [#28593](https://github.com/cypress-io/cypress/pull/28593).
- Fixed a regression in [`13.6.2`](https://docs.cypress.io/guides/references/changelog/13.6.2) where the `body` element was not highlighted correctly in Test Replay. Fixed in [#28627](https://github.com/cypress-io/cypress/pull/28627).
- Fixed an issue where some cross-origin logs, like assertions or cy.clock(), were getting too many dom snapshots. Fixes [#28609](https://github.com/cypress-io/cypress/issues/28609).
- Fixed asset capture for Test Replay for requests that are routed through service workers. This addresses an issue where styles were not being applied properly in Test Replay and `cy.intercept` was not working properly for requests in this scenario. Fixes [#28516](https://github.com/cypress-io/cypress/issues/28516).

**Performance:**

Expand Down
60 changes: 59 additions & 1 deletion packages/proxy/lib/http/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,19 @@ import type { RemoteStates } from '@packages/server/lib/remote_states'
import type { CookieJar, SerializableAutomationCookie } from '@packages/server/lib/util/cookies'
import type { ResourceTypeAndCredentialManager } from '@packages/server/lib/util/resourceTypeAndCredentialManager'
import type { ProtocolManagerShape } from '@packages/types'
import type Protocol from 'devtools-protocol'
import { ServiceWorkerManager } from './util/service-worker-manager'

function getRandomColorFn () {
return chalk.hex(`#${Number(
Math.floor(Math.random() * 0xFFFFFF),
).toString(16).padStart(6, 'F').toUpperCase()}`)
}

const hasServiceWorkerHeader = (headers: Record<string, string | string[] | undefined>) => {
return headers?.['service-worker'] === 'script' || headers?.['Service-Worker'] === 'script'
}

export const isVerboseTelemetry = true

const isVerbose = isVerboseTelemetry
Expand Down Expand Up @@ -273,6 +279,7 @@ export class Http {
autUrl?: string
getCookieJar: () => CookieJar
protocolManager?: ProtocolManagerShape
serviceWorkerManager: ServiceWorkerManager = new ServiceWorkerManager()

constructor (opts: ServerCtx & { middleware?: HttpMiddlewareStacks }) {
this.buffers = new HttpBuffers()
Expand Down Expand Up @@ -332,6 +339,18 @@ export class Http {
getAUTUrl: this.getAUTUrl,
setAUTUrl: this.setAUTUrl,
getPreRequest: (cb) => {
// The initial request that loads the service worker does not always get sent to CDP. Thus, we need to explicitly ignore it. We determine
// it's the service worker request via the `service-worker` header
if (hasServiceWorkerHeader(req.headers)) {
ctx.debug('Ignoring service worker script since we are not guaranteed to receive it', req.proxiedUrl)

cb({
noPreRequestExpected: true,
})

return
}

return this.preRequests.get(ctx.req, ctx.debug, cb)
},
addPendingUrlWithoutPreRequest: (url) => {
Expand Down Expand Up @@ -429,20 +448,28 @@ export class Http {
}
}

reset (options: { resetPreRequests: boolean }) {
reset (options: { resetPreRequests: boolean, resetBetweenSpecs: boolean }) {
this.buffers.reset()
this.setAUTUrl(undefined)

if (options.resetPreRequests) {
this.preRequests.reset()
}

if (options.resetBetweenSpecs) {
this.serviceWorkerManager = new ServiceWorkerManager()
}
}

setBuffer (buffer) {
return this.buffers.set(buffer)
}

addPendingBrowserPreRequest (browserPreRequest: BrowserPreRequest) {
if (this.shouldIgnorePendingRequest(browserPreRequest)) {
return
}

this.preRequests.addPending(browserPreRequest)
}

Expand All @@ -454,6 +481,18 @@ export class Http {
this.preRequests.addPendingUrlWithoutPreRequest(url)
}

updateServiceWorkerRegistrations (data: Protocol.ServiceWorker.WorkerRegistrationUpdatedEvent) {
this.serviceWorkerManager.updateServiceWorkerRegistrations(data)
}

updateServiceWorkerVersions (data: Protocol.ServiceWorker.WorkerVersionUpdatedEvent) {
this.serviceWorkerManager.updateServiceWorkerVersions(data)
}

updateServiceWorkerClientSideRegistrations (data: { scriptURL: string, initiatorURL: string }) {
this.serviceWorkerManager.addInitiatorToServiceWorker({ scriptURL: data.scriptURL, initiatorURL: data.initiatorURL })
}

setProtocolManager (protocolManager: ProtocolManagerShape) {
this.protocolManager = protocolManager
this.preRequests.setProtocolManager(protocolManager)
Expand All @@ -462,4 +501,23 @@ export class Http {
setPreRequestTimeout (timeout: number) {
this.preRequests.setPreRequestTimeout(timeout)
}

private shouldIgnorePendingRequest (browserPreRequest: BrowserPreRequest) {
// The initial request that loads the service worker does not always get sent to CDP. If it does, we want it to not clog up either the prerequests
// or pending requests. Thus, we need to explicitly ignore it here and in `get`. We determine it's the service worker request via the
// `service-worker` header
if (hasServiceWorkerHeader(browserPreRequest.headers)) {
debugVerbose('Ignoring service worker script since we are not guaranteed to receive it: %o', browserPreRequest)

return true
}

if (this.serviceWorkerManager.processBrowserPreRequest(browserPreRequest)) {
debugVerbose('Not correlating request since it is fully controlled by the service worker and the correlation will happen within the service worker: %o', browserPreRequest)

return true
}

return false
}
}
17 changes: 3 additions & 14 deletions packages/proxy/lib/http/util/prerequests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,9 @@ export class PreRequests {
}

addPending (browserPreRequest: BrowserPreRequest) {
metrics.browserPreRequestsReceived++
const key = `${browserPreRequest.method}-${tryDecodeURI(browserPreRequest.url)}`

metrics.browserPreRequestsReceived++
const pendingRequest = this.pendingRequests.shift(key)

if (pendingRequest) {
Expand Down Expand Up @@ -230,19 +231,6 @@ export class PreRequests {
}

get (req: CypressIncomingRequest, ctxDebug, callback: GetPreRequestCb) {
// The initial request that loads the service worker does not get sent to CDP and it happens prior
// to the service worker target being added. Thus, we need to explicitly ignore it. We determine
// it's the service worker request via the `sec-fetch-dest` header
if (req.headers['sec-fetch-dest'] === 'serviceworker') {
ctxDebug('Ignoring request with sec-fetch-dest: serviceworker', req.proxiedUrl)

callback({
noPreRequestExpected: true,
})

return
}

const proxyRequestReceivedTimestamp = performance.now() + performance.timeOrigin

metrics.proxyRequestsReceived++
Expand All @@ -252,6 +240,7 @@ export class PreRequests {
if (pendingPreRequest) {
metrics.immediatelyMatchedRequests++
ctxDebug('Incoming request %s matches known pre-request: %o', key, pendingPreRequest)

callback({
browserPreRequest: {
...pendingPreRequest.browserPreRequest,
Expand Down
186 changes: 186 additions & 0 deletions packages/proxy/lib/http/util/service-worker-manager.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
import Debug from 'debug'
import type { BrowserPreRequest } from '../../types'
import type Protocol from 'devtools-protocol'

const debug = Debug('cypress:proxy:service-worker-manager')

type ServiceWorkerRegistration = {
registrationId: string
scopeURL: string
activatedServiceWorker?: ServiceWorker
}

type ServiceWorker = {
registrationId: string
scriptURL: string
initiatorURL?: string
controlledURLs: Set<string>
}

type RegisterServiceWorkerOptions = {
registrationId: string
scopeURL: string
}

type UnregisterServiceWorkerOptions = {
registrationId: string
}

type AddActivatedServiceWorkerOptions = {
registrationId: string
scriptURL: string
}

type AddInitiatorToServiceWorkerOptions = {
scriptURL: string
initiatorURL: string
}

/**
* Manages service worker registrations and their controlled URLs.
*
* The basic lifecycle is as follows:
*
* 1. A service worker is registered via `registerServiceWorker`.
* 2. The service worker is activated via `addActivatedServiceWorker`.
*
* At some point while 1 and 2 are happening:
*
* 3. We receive a message from the browser that a service worker has been initiated with the `addInitiatorToServiceWorker` method.
*
* At this point, when the manager tries to process a browser pre-request, it will check if the request is controlled by a service worker.
* It determines it is controlled by a service worker if:
*
* 1. The document URL for the browser pre-request matches the initiator URL for the service worker.
* 2. The request URL is within the scope of the service worker or the request URL's initiator is controlled by the service worker.
*/
export class ServiceWorkerManager {
private serviceWorkerRegistrations: Map<string, ServiceWorkerRegistration> = new Map<string, ServiceWorkerRegistration>()
private pendingInitiators: Map<string, string> = new Map<string, string>()

/**
* Goes through the list of service worker registrations and adds or removes them from the manager.
*/
updateServiceWorkerRegistrations (data: Protocol.ServiceWorker.WorkerRegistrationUpdatedEvent) {
data.registrations.forEach((registration) => {
if (registration.isDeleted) {
this.unregisterServiceWorker({ registrationId: registration.registrationId })
} else {
this.registerServiceWorker({ registrationId: registration.registrationId, scopeURL: registration.scopeURL })
}
})
}

/**
* Goes through the list of service worker versions and adds any that are activated to the manager.
*/
updateServiceWorkerVersions (data: Protocol.ServiceWorker.WorkerVersionUpdatedEvent) {
data.versions.forEach((version) => {
if (version.status === 'activated') {
this.addActivatedServiceWorker({ registrationId: version.registrationId, scriptURL: version.scriptURL })
}
})
}

/**
* Adds an initiator URL to a service worker. If the service worker has not yet been activated, the initiator URL is added to a pending list and will
* be added to the service worker when it is activated.
*/
addInitiatorToServiceWorker ({ scriptURL, initiatorURL }: AddInitiatorToServiceWorkerOptions) {
let initiatorAdded = false

for (const registration of this.serviceWorkerRegistrations.values()) {
if (registration.activatedServiceWorker?.scriptURL === scriptURL) {
registration.activatedServiceWorker.initiatorURL = initiatorURL

initiatorAdded = true
break
}
}

if (!initiatorAdded) {
this.pendingInitiators.set(scriptURL, initiatorURL)
}
}

/**
* Processes a browser pre-request to determine if it is controlled by a service worker. If it is, the service worker's controlled URLs are updated with the given request URL.
*
* @param browserPreRequest The browser pre-request to process.
* @returns `true` if the request is controlled by a service worker, `false` otherwise.
*/
processBrowserPreRequest (browserPreRequest: BrowserPreRequest) {
if (browserPreRequest.initiator?.type === 'preload') {
return false
}

let requestControlledByServiceWorker = false

this.serviceWorkerRegistrations.forEach((registration) => {
const activatedServiceWorker = registration.activatedServiceWorker
const paramlessDocumentURL = browserPreRequest.documentURL.split('?')[0]

if (!activatedServiceWorker || activatedServiceWorker.initiatorURL !== paramlessDocumentURL) {
return
}

const paramlessURL = browserPreRequest.url.split('?')[0]
const paramlessInitiatorURL = browserPreRequest.initiator?.url?.split('?')[0]
const paramlessCallStackURL = browserPreRequest.initiator?.stack?.callFrames[0]?.url?.split('?')[0]
const urlIsControlled = paramlessURL.startsWith(registration.scopeURL)
const initiatorUrlIsControlled = paramlessInitiatorURL && activatedServiceWorker.controlledURLs?.has(paramlessInitiatorURL)
const topStackUrlIsControlled = paramlessCallStackURL && activatedServiceWorker.controlledURLs?.has(paramlessCallStackURL)

if (urlIsControlled || initiatorUrlIsControlled || topStackUrlIsControlled) {
activatedServiceWorker.controlledURLs.add(paramlessURL)
requestControlledByServiceWorker = true
}
})

return requestControlledByServiceWorker
}

/**
* Registers the given service worker with the given scope. Will not overwrite an existing registration.
*/
private registerServiceWorker ({ registrationId, scopeURL }: RegisterServiceWorkerOptions) {
// Only register service workers if they haven't already been registered
if (this.serviceWorkerRegistrations.get(registrationId)?.scopeURL === scopeURL) {
return
}

this.serviceWorkerRegistrations.set(registrationId, {
registrationId,
scopeURL,
})
}

/**
* Unregisters the service worker with the given registration ID.
*/
private unregisterServiceWorker ({ registrationId }: UnregisterServiceWorkerOptions) {
this.serviceWorkerRegistrations.delete(registrationId)
}

/**
* Adds an activated service worker to the manager.
*/
private addActivatedServiceWorker ({ registrationId, scriptURL }: AddActivatedServiceWorkerOptions) {
const registration = this.serviceWorkerRegistrations.get(registrationId)

if (registration) {
const initiatorURL = this.pendingInitiators.get(scriptURL)

registration.activatedServiceWorker = {
registrationId,
scriptURL,
controlledURLs: new Set<string>(),
initiatorURL: initiatorURL || registration.activatedServiceWorker?.initiatorURL,
}

this.pendingInitiators.delete(scriptURL)
} else {
debug('Could not find service worker registration for registration ID %s', registrationId)
}
}
}
Loading

0 comments on commit c6f5e9a

Please sign in to comment.