diff --git a/mypy/build.py b/mypy/build.py
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -1940,6 +1940,8 @@ def __init__(
                 raise
             if follow_imports == "silent":
                 self.ignore_all = True
+        elif path and is_silent_import_module(manager, path):
+            self.ignore_all = True
         self.path = path
         if path:
             self.abspath = os.path.abspath(path)
@@ -2613,11 +2615,8 @@ def find_module_and_diagnose(
                 else:
                     skipping_module(manager, caller_line, caller_state, id, result)
             raise ModuleNotFound
-        if not manager.options.no_silence_site_packages:
-            for dir in manager.search_paths.package_path + manager.search_paths.typeshed_path:
-                if is_sub_path(result, dir):
-                    # Silence errors in site-package dirs and typeshed
-                    follow_imports = "silent"
+        if is_silent_import_module(manager, result):
+            follow_imports = "silent"
         return (result, follow_imports)
     else:
         # Could not find a module.  Typically the reason is a
@@ -3560,3 +3559,12 @@ def record_missing_stub_packages(cache_dir: str, missing_stub_packages: set[str]
     else:
         if os.path.isfile(fnam):
             os.remove(fnam)
+
+
+def is_silent_import_module(manager: BuildManager, path: str) -> bool:
+    if not manager.options.no_silence_site_packages:
+        for dir in manager.search_paths.package_path + manager.search_paths.typeshed_path:
+            if is_sub_path(path, dir):
+                # Silence errors in site-package dirs and typeshed
+                return True
+    return False
diff --git a/mypy/server/update.py b/mypy/server/update.py
--- a/mypy/server/update.py
+++ b/mypy/server/update.py
@@ -963,9 +963,10 @@ def key(node: FineGrainedDeferredNode) -> int:
 
     nodes = sorted(nodeset, key=key)
 
-    options = graph[module_id].options
+    state = graph[module_id]
+    options = state.options
     manager.errors.set_file_ignored_lines(
-        file_node.path, file_node.ignored_lines, options.ignore_errors
+        file_node.path, file_node.ignored_lines, options.ignore_errors or state.ignore_all
     )
 
     targets = set()

diff --git a/test-data/unit/fine-grained-modules.test b/test-data/unit/fine-grained-modules.test
--- a/test-data/unit/fine-grained-modules.test
+++ b/test-data/unit/fine-grained-modules.test
@@ -2206,3 +2206,41 @@ a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missin
 a.py:1: error: Library stubs not installed for "jack"
 a.py:1: note: Hint: "python3 -m pip install types-JACK-Client"
 a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+
+[case testIgnoreErrorsFromTypeshed]
+# flags: --custom-typeshed-dir tmp/ts --follow-imports=normal
+# cmd1: mypy a.py
+# cmd2: mypy a.py
+
+[file a.py]
+import foobar
+
+[file ts/stdlib/abc.pyi]
+[file ts/stdlib/builtins.pyi]
+class object: pass
+class str: pass
+class ellipsis: pass
+[file ts/stdlib/sys.pyi]
+[file ts/stdlib/types.pyi]
+[file ts/stdlib/typing.pyi]
+def cast(x): ...
+[file ts/stdlib/typing_extensions.pyi]
+[file ts/stdlib/VERSIONS]
+[file ts/stubs/mypy_extensions/mypy_extensions.pyi]
+
+[file ts/stdlib/foobar.pyi.2]
+# We report no errors from typeshed. It would be better to test ignoring
+# errors from PEP 561 packages, but it's harder to test and uses the
+# same code paths, so we are using typeshed instead.
+import baz
+import zar
+undefined
+
+[file ts/stdlib/baz.pyi.2]
+import whatever
+undefined
+
+[out]
+a.py:1: error: Cannot find implementation or library stub for module named "foobar"
+a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+==

Errors are shown in third-party packages with inline annotations when using daemon
To reproduce:
* Start daemon on a code that uses e.g. `pytest` before installing `pytest`
* Install `pytest` (it has inline annotations)
* Run `dmypy run` again and get a lot of error messages.

Note that restarting the daemon fixes the issue.
This also happens with `types-*` packages. For example:
* Start daemon on a file that imports `requests` using `--disallow-untyped-defs`
* Install `types-requests`
* Run `dmypy run` and get lots of errors 

Again restarting the daemon helps.
mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddFile mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeletionTriggersImport mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testRemoveSubmoduleFromBuild1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddPackage5 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeletionOfSubmoduleTriggersImportFrom1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportLineNumber1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddFileWithErrors mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddPackage6 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportLineNumber2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddFileFixesError mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddPackage7 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddPackage1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeletionOfSubmoduleTriggersImportFrom2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddPackage8 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddPackage2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddFileFixesAndGeneratesError1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeletionOfSubmoduleTriggersImport mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteBasic mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddPackage3 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddFileFixesAndGeneratesError2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteSubpackageWithNontrivialParent1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteDepOfDunderInit1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteDepOfDunderInit2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteModuleWithError mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddFileGeneratesError1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddPackage4 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteModuleWithErrorInsidePackage mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeletionTriggersImportFrom mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddNonPackageSubdir mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddFilePreservesError1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testModifyTwoFilesNoError1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddFileWhichImportsLibModuleWithErrors mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddTwoFilesNoError mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddFilePreservesError2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteSubpackageInit1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testRenameModule mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddTwoFilesErrorsInBoth mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddSubpackageInit2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteFileWSuperClass mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testModifyTwoFilesNoError2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddTwoFilesErrorsElsewhere mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeletePackage3 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteTwoFilesErrorsElsewhere mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteFileWithinPackage mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testModifyTwoFilesErrorsElsewhere mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteTwoFilesNoErrors mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testModifyTwoFilesErrorsInBoth mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeletePackage1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeletePackage4 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testModifyTwoFilesFixErrorsInBoth mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeletePackage2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteTwoFilesFixErrors mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeletePackage5 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testRefreshImportOfIgnoredModule1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testSkipImportsWithinPackage mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeletePackage6 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddFileWhichImportsLibModule mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteModuleWithinPackageInitIgnored mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testRefreshImportOfIgnoredModule2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testIncrementalWithIgnoresTwice mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testIgnoreMissingImports mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddImport mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarPropagateChange2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testIgnoredImport2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testSkipImports mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddImport2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteIndirectDependency mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarAddMissingDependency1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testChangeModuleToVariable mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testRefreshImportIfMypyElse1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarAddMissingDependencyInsidePackage1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarAddMissingDependency2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testChangeVariableToModule mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarPropagateChange1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarAddMissingDependencyInsidePackage2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarWithinFunction mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarOverlap2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarRemoveDependency1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarAddMissingDependencyWithinClass mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarOverlap3 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddAndUseClass1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarSomethingMoved mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarRemoveDependency2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportPartialAssign mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddAndUseClass2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportStarOverlap mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddAndUseClass4 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDeleteFileWithErrors mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testSkipButDontIgnore1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddAndUseClass3 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddAndUseClass4_2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testErrorButDontIgnore1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testSkipButDontIgnore2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testAddAndUseClass5 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testErrorButDontIgnore2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testTurnPackageToModule mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testErrorButDontIgnore4 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testImportedMissingSubpackage mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineAddedMissingStubs mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testModuleToPackage mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineAddedMissingStubsPackage mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testQualifiedSubpackage1 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineAddedMissingStubsPackageFrom mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineAddedMissingStubsPackagePartial mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineAddedMissingStubsIgnore mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineAddedMissingStubsPackagePartialGetAttr mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineAddedMissingStubsIgnorePackage mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineFollowImportSkipNotInvalidatedOnPresentPackage mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineAddedMissingStubsIgnorePackagePartial mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineAddedMissingStubsIgnorePackageFrom mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineFollowImportSkipNotInvalidatedOnPresent mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testMissingStubAdded2 mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testIgnoredAttrReprocessedModule mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineFollowImportSkipNotInvalidatedOnAdded mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testFineAddedSkippedStubsPackageFrom mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testDoNotFollowImportToNonStubFile mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testLibraryStubsNotInstalled mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testMissingStubAdded1
mypy/test/testfinegrained.py::FineGrainedSuite::fine-grained-modules.test::testIgnoreErrorsFromTypeshed