diff --git a/lib/handler/retry-handler.js b/lib/handler/retry-handler.js
--- a/lib/handler/retry-handler.js
+++ b/lib/handler/retry-handler.js
@@ -242,14 +242,12 @@ class RetryHandler {
         }
 
         const { start, size, end = size } = range
-
         assert(
-          start != null && Number.isFinite(start) && this.start !== start,
+          start != null && Number.isFinite(start),
           'content-range mismatch'
         )
-        assert(Number.isFinite(start))
         assert(
-          end != null && Number.isFinite(end) && this.end !== end,
+          end != null && Number.isFinite(end),
           'invalid content-length'
         )
 

diff --git a/test/retry-handler.js b/test/retry-handler.js
--- a/test/retry-handler.js
+++ b/test/retry-handler.js
@@ -876,3 +876,106 @@ test('Should be able to properly pass the minTimeout to the RetryContext when co
 
   await t.completed
 })
+test('Issue#2986 - Handle custom 206', { only: true }, async t => {
+  t = tspl(t, { plan: 8 })
+
+  const chunks = []
+  let counter = 0
+
+  // Took from: https://github.com/nxtedition/nxt-lib/blob/4b001ebc2f22cf735a398f35ff800dd553fe5933/test/undici/retry.js#L47
+  let x = 0
+  const server = createServer((req, res) => {
+    if (x === 0) {
+      t.deepStrictEqual(req.headers.range, 'bytes=0-3')
+      res.setHeader('etag', 'asd')
+      res.write('abc')
+      setTimeout(() => {
+        res.destroy()
+      }, 1e2)
+    } else if (x === 1) {
+      t.deepStrictEqual(req.headers.range, 'bytes=3-')
+      res.setHeader('content-range', 'bytes 3-6/6')
+      res.setHeader('etag', 'asd')
+      res.statusCode = 206
+      res.end('def')
+    }
+    x++
+  })
+
+  const dispatchOptions = {
+    retryOptions: {
+      retry: function (err, _, done) {
+        counter++
+
+        if (err.code && err.code === 'UND_ERR_DESTROYED') {
+          return done(false)
+        }
+
+        if (err.statusCode === 206) return done(err)
+
+        setTimeout(done, 800)
+      }
+    },
+    method: 'GET',
+    path: '/',
+    headers: {
+      'content-type': 'application/json'
+    }
+  }
+
+  server.listen(0, () => {
+    const client = new Client(`http://localhost:${server.address().port}`)
+    const handler = new RetryHandler(dispatchOptions, {
+      dispatch: (...args) => {
+        return client.dispatch(...args)
+      },
+      handler: {
+        onRequestSent () {
+          t.ok(true, 'pass')
+        },
+        onConnect () {
+          t.ok(true, 'pass')
+        },
+        onBodySent () {
+          t.ok(true, 'pass')
+        },
+        onHeaders (status, _rawHeaders, resume, _statusMessage) {
+          t.strictEqual(status, 200)
+          return true
+        },
+        onData (chunk) {
+          chunks.push(chunk)
+          return true
+        },
+        onComplete () {
+          t.strictEqual(Buffer.concat(chunks).toString('utf-8'), 'abcdef')
+          t.strictEqual(counter, 1)
+        },
+        onError () {
+          t.fail()
+        }
+      }
+    })
+
+    client.dispatch(
+      {
+        method: 'GET',
+        path: '/',
+        headers: {
+          'content-type': 'application/json',
+          Range: 'bytes=0-3'
+        }
+      },
+      handler
+    )
+
+    after(async () => {
+      await client.close()
+
+      server.close()
+      await once(server, 'close')
+    })
+  })
+
+  await t.completed
+})

Fails to retrieve the first piece of data when downloading partial content with `RetryAgent`
## Bug Description

<!-- A clear and concise description of what the bug is. -->

When trying to download the file in parts (using `RetryAgent`), the download of the first part (when first byte of range is equal to zero) fails due to a `content-range mismatch` error inside `RetryHandler` `onHeaders` method.

## Reproducible By

<!-- A step by step list on how the bug can be reproduced for examination. -->

Here's a simple example of reproducing the issue:
```js
const { setGlobalDispatcher, RetryAgent, Agent, request } = require('undici');

setGlobalDispatcher(new RetryAgent(new Agent()));

const start = 0;
const end = 26664011;
const size = end - start + 1;
const url =
  'https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/MI201109210084_mpeg-4_hd_high_1080p25_10mbits.mp4';
const options = { headers: { Range: `bytes=${start}-${end}` } };

request(url, options)
  .then((response) => {
    const contentLength = Number(response.headers['content-length']);
    console.log(contentLength === size);
  })
  .catch((e) => {
    console.log(e.message === 'content-range mismatch'); // true
  });
```

## Expected Behavior

<!-- A clear and concise description of what you expected to happen. -->

I expect the request to successfully execute and return the first piece of data. If you remove `RetryAgent` from the code example above, the request will succeed:

```js
const { setGlobalDispatcher, Agent, request } = require('undici');

setGlobalDispatcher(new Agent());

const start = 0;
const end = 26664011;
const size = end - start + 1;
const url =
  'https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/MI201109210084_mpeg-4_hd_high_1080p25_10mbits.mp4';
const options = { headers: { Range: `bytes=${start}-${end}` } };

request(url, options)
  .then((response) => {
    const contentLength = Number(response.headers['content-length']);
    console.log(contentLength === size); // true
  })
  .catch((e) => {
    console.log(e.message === 'content-range mismatch');
  });
```

## Logs & Screenshots

<!-- If applicable, add screenshots to help explain your problem, or 
alternatively add your console logs here. -->

```
AssertionError [ERR_ASSERTION]: content-range mismatch
    at RetryHandler.onHeaders (/Users/user/project/node_modules/undici/lib/handler/retry-handler.js:248:9)
    at Request.onHeaders (/Users/user/project/node_modules/undici/lib/core/request.js:243:29)
    at Parser.onHeadersComplete (/Users/user/project/node_modules/undici/lib/dispatcher/client-h1.js:507:27)
    at wasm_on_headers_complete (/Users/user/project/node_modules/undici/lib/dispatcher/client-h1.js:121:30)
    at wasm://wasm/0003626a:wasm-function[11]:0x494
    at wasm://wasm/0003626a:wasm-function[51]:0x1003
    at wasm://wasm/0003626a:wasm-function[68]:0x6e8e
    at wasm://wasm/0003626a:wasm-function[67]:0x1568
    at wasm://wasm/0003626a:wasm-function[21]:0x552
    at Parser.execute (/Users/user/project/node_modules/undici/lib/dispatcher/client-h1.js:262:22) {
  generatedMessage: false,
  code: 'ERR_ASSERTION',
  actual: false,
  expected: true,
  operator: '=='
}
```

## Environment

<!-- This is just your OS and environment information [e.g. Ubuntu 18.04 LTS,
Node v14.14.0] -->
macOS Sonoma 14.3.1 (23D60)
Node v20.11.1

