fix(core): prioritize nxignore for watcher updates (#20975)

packages/nx/Cargo.toml

@@ -8,11 +8,12 @@ anyhow = "1.0.71"
 colored = "2"
 crossbeam-channel = '0.5'
 dashmap = { version = "5.5.3", features = ["rayon"] }
+dunce = "1"
 fs_extra = "1.3.0"
 globset = "0.4.10"
 hashbrown = { version = "0.14.3", features = ["rayon", "rkyv"] }
 ignore = '0.4'
-ignore-files = "1.3.0"
+ignore-files = "2.0.0"
 itertools = "0.10.5"
 once_cell = "1.18.0"
 parking_lot = { version = "0.12.1", features = ["send_guard"] }
@@ -31,10 +32,10 @@ tokio = { version = "1.28.2", features = ["fs"] }
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
 walkdir = '2.3.3'
-watchexec = "2.3.0"
+watchexec = "3.0.1"
-watchexec-events = "1.0.0"
+watchexec-events = "2.0.1"
-watchexec-filterer-ignore = "1.2.1"
+watchexec-filterer-ignore = "3.0.0"
-watchexec-signals = "1.0.0"
+watchexec-signals = "2.1.0"
 xxhash-rust = { version = '0.8.5', features = ['xxh3', 'xxh64'] }
 swc_common = "0.31.16"
 swc_ecma_parser = { version = "0.137.1", features = ["typescript"] }

packages/nx/src/native/tests/watcher.spec.ts

@@ -8,11 +8,14 @@ describe('watcher', () => {
     temp = new TempFs('watch-dir');
     temp.createFilesSync({
       '.gitignore': 'node_modules/\n.env.local',
-      '.nxignore': 'app2/\n!.env.local',
+      '.nxignore': 'app2/\n!.env.*\nboo.txt',
       '.env.local': '',
       'app1/main.js': '',
       'app1/main.css': '',
       'app2/main.js': '',
+      'inner/.gitignore': '.env.inner',
+      'inner/boo.txt': '',
+      'inner/.env.inner': '',
       'nested-ignore/.gitignore': '*',
       'nested-ignore/file.js': '',
       'node_modules/module/index.js': '',
@@ -51,7 +54,7 @@ describe('watcher', () => {
       await wait();
       temp.createFileSync('app1/main.html', JSON.stringify({}));
     });
-  }, 10000);
+  }, 15000);
 
   it('should trigger the callback when files are updated', async () => {
     return new Promise<void>(async (done) => {
@@ -76,7 +79,7 @@ describe('watcher', () => {
       await wait();
       temp.appendFile('app1/main.js', 'update');
     });
-  }, 10000);
+  }, 15000);
 
   it('should watch file renames', async () => {
     return new Promise<void>(async (done) => {
@@ -103,7 +106,7 @@ describe('watcher', () => {
       await wait();
       temp.renameFile('app1/main.js', 'app1/rename.js');
     });
-  }, 10000);
+  }, 15000);
 
   it('should trigger on deletes', async () => {
     return new Promise<void>(async (done) => {
@@ -125,7 +128,7 @@ describe('watcher', () => {
       await wait();
       temp.removeFileSync('app1/main.js');
     });
-  }, 10000);
+  }, 15000);
 
   it('should ignore nested gitignores', async () => {
     return new Promise<void>(async (done) => {
@@ -137,7 +140,7 @@ describe('watcher', () => {
         expect(paths).toMatchInlineSnapshot(`
         [
           {
-            "path": "boo.txt",
+            "path": "bar.txt",
             "type": "create",
           },
         ]
@@ -149,21 +152,27 @@ describe('watcher', () => {
       // should not be triggered
       temp.createFileSync('nested-ignore/hello1.txt', '');
       await wait();
-      temp.createFileSync('boo.txt', '');
+      temp.createFileSync('bar.txt', '');
     });
-  }, 10000);
+  }, 15000);
 
-  it('should include files that are negated in nxignore but are ignored in gitignore', async () => {
+  it('prioritize nxignore over gitignores', async () => {
     return new Promise<void>(async (done) => {
       await wait();
       watcher = new Watcher(temp.tempDir);
       watcher.watch((err, paths) => {
         expect(paths.some(({ path }) => path === '.env.local')).toBeTruthy();
+        expect(
+          paths.some(({ path }) => path === 'inner/.env.inner')
+        ).toBeTruthy();
+        expect(paths.some(({ path }) => path === 'inner/boo.txt')).toBeFalsy();
         done();
       });
 
       await wait(2000);
       temp.appendFile('.env.local', 'hello');
+      temp.appendFile('inner/.env.inner', 'hello');
+      temp.appendFile('inner/boo.txt', 'hello');
     });
   }, 15000);
 });

packages/nx/src/native/watch/mod.rs

@@ -1,5 +1,4 @@
 mod types;
 mod utils;
-mod watch_config;
 mod watch_filterer;
 mod watcher;

packages/nx/src/native/watch/watch_config.rs

@@ -1,54 +0,0 @@
-use std::sync::Arc;
-
-use ignore_files::IgnoreFilter;
-use tracing::trace;
-use watchexec::config::RuntimeConfig;
-use watchexec_filterer_ignore::IgnoreFilterer;
-
-use crate::native::watch::utils::{get_ignore_files, get_nx_ignore};
-use crate::native::watch::watch_filterer::WatchFilterer;
-
-pub(super) async fn create_runtime(
-    origin: &str,
-    additional_globs: &[&str],
-    use_ignore: bool,
-) -> napi::Result<RuntimeConfig> {
-    let ignore_files = get_ignore_files(use_ignore, origin);
-    let nx_ignore_file = get_nx_ignore(origin);
-
-    trace!(
-        ?use_ignore,
-        ?additional_globs,
-        ?ignore_files,
-        "Using these ignore files for the watcher"
-    );
-    let mut filter = if let Some(ignore_files) = ignore_files {
-        IgnoreFilter::new(origin, &ignore_files)
-            .await
-            .map_err(anyhow::Error::from)?
-    } else {
-        IgnoreFilter::empty(origin)
-    };
-
-    filter
-        .add_globs(additional_globs, Some(&origin.into()))
-        .map_err(anyhow::Error::from)?;
-
-    // always add the .nxignore file after all other ignores are loaded so that it has the highest priority
-    if let Some(nx_ignore_file) = nx_ignore_file {
-        filter
-            .add_file(&nx_ignore_file)
-            .await
-            .map_err(anyhow::Error::from)?;
-    }
-
-    let mut runtime = RuntimeConfig::default();
-    runtime.filterer(Arc::new(WatchFilterer {
-        inner: IgnoreFilterer(filter),
-    }));
-
-    // let watch_directories = get_watch_directories(origin);
-    // trace!(directories = ?watch_directories, "watching");
-    runtime.pathset([&origin]);
-    Ok(runtime)
-}

packages/nx/src/native/watch/watch_filterer.rs

@@ -1,16 +1,53 @@
+use ignore::Match;
 use tracing::trace;
 use watchexec::error::RuntimeError;
 use watchexec::filter::Filterer;
 use watchexec_events::filekind::{CreateKind, FileEventKind, ModifyKind, RemoveKind};
 
+use ignore_files::IgnoreFilter;
 use watchexec_events::{Event, FileType, Priority, Source, Tag};
 use watchexec_filterer_ignore::IgnoreFilterer;
 
-use crate::native::watch::utils::transform_event;
+use crate::native::watch::utils::{get_ignore_files, get_nx_ignore, transform_event};
 
 #[derive(Debug)]
 pub struct WatchFilterer {
-    pub inner: IgnoreFilterer,
+    pub nx_ignore: Option<IgnoreFilter>,
+    pub git_ignore: IgnoreFilterer,
+}
+
+impl WatchFilterer {
+    fn filter_event(&self, event: &Event, priority: Priority) -> bool {
+        let mut pass = true;
+        for (path, file_type) in event.paths() {
+            let path = dunce::simplified(path);
+            let is_dir = file_type.map_or(false, |t| matches!(t, FileType::Dir));
+            let nx_ignore_match_type = if let Some(nx_ignore) = &self.nx_ignore {
+                nx_ignore.match_path(path, is_dir)
+            } else {
+                Match::None
+            };
+
+            // if the nxignore file contains this file as a whitelist,
+            // we do not want gitignore to filter it out, so it will always pass as true
+            if matches!(nx_ignore_match_type, Match::Whitelist(_)) {
+                trace!(?path, "nxignore whitelist match, ignoring gitignore");
+                pass &= true;
+            // If the nxignore file contains this file as an ignore,
+            // then there's no point in checking the gitignore file
+            } else if matches!(nx_ignore_match_type, Match::Ignore(_)) {
+                trace!(?path, "nxignore ignore match, ignoring gitignore");
+                pass &= false;
+            } else {
+                pass &= self
+                    .git_ignore
+                    .check_event(event, priority)
+                    .expect("git ignore check never errors")
+            }
+        }
+
+        pass
+    }
 }
 
 /// Used to filter out events that that come from watchexec
@@ -19,12 +56,11 @@ impl Filterer for WatchFilterer {
         let transformed = transform_event(watch_event);
         let event = transformed.as_ref().unwrap_or(watch_event);
 
-        if !self.inner.check_event(event, priority)? {
+        trace!(?event, "checking if event is valid");
+        if !self.filter_event(event, priority) {
             return Ok(false);
         }
 
-        trace!(?event, "checking if event is valid");
-
         //
         // Tags will be a Vec that contains multiple types of information for a given event
         // We are only interested if:
@@ -67,3 +103,51 @@ impl Filterer for WatchFilterer {
         Ok(true)
     }
 }
+
+pub(super) async fn create_filter(
+    origin: &str,
+    additional_globs: &[String],
+    use_ignore: bool,
+) -> anyhow::Result<WatchFilterer> {
+    let ignore_files = get_ignore_files(use_ignore, origin);
+    let nx_ignore_file = get_nx_ignore(origin);
+
+    trace!(
+        ?use_ignore,
+        ?additional_globs,
+        ?ignore_files,
+        "Using these ignore files for the watcher"
+    );
+    let mut git_ignore = if let Some(ignore_files) = ignore_files {
+        IgnoreFilter::new(origin, &ignore_files)
+            .await
+            .map_err(anyhow::Error::from)?
+    } else {
+        IgnoreFilter::empty(origin)
+    };
+
+    git_ignore
+        .add_globs(
+            &additional_globs
+                .iter()
+                .map(String::as_ref)
+                .collect::<Vec<_>>(),
+            Some(&origin.into()),
+        )
+        .map_err(anyhow::Error::from)?;
+
+    let nx_ignore = if let Some(nx_ignore_file) = nx_ignore_file {
+        Some(
+            IgnoreFilter::new(origin, &[nx_ignore_file])
+                .await
+                .map_err(anyhow::Error::from)?,
+        )
+    } else {
+        None
+    };
+
+    Ok(WatchFilterer {
+        git_ignore: IgnoreFilterer(git_ignore),
+        nx_ignore,
+    })
+}

packages/nx/src/native/watch/watcher.rs

@@ -1,10 +1,10 @@
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
-use std::convert::Infallible;
 use std::path::MAIN_SEPARATOR;
 use std::sync::Arc;
 
 use crate::native::watch::types::{EventType, WatchEvent, WatchEventInternal};
+use crate::native::watch::watch_filterer;
 use napi::bindgen_prelude::*;
 use napi::threadsafe_function::{
     ThreadSafeCallContext, ThreadsafeFunction, ThreadsafeFunctionCallMode,
@@ -13,15 +13,10 @@ use napi::{Env, JsFunction, JsObject};
 use rayon::prelude::*;
 use tracing::trace;
 use tracing_subscriber::EnvFilter;
-use watchexec::action::{Action, Outcome};
-use watchexec::config::{InitConfig, RuntimeConfig};
-use watchexec::event::Tag;
 use watchexec::Watchexec;
-use watchexec_events::{Event, Keyboard, Priority};
+use watchexec_events::{Event, Priority, Tag};
 use watchexec_signals::Signal;
 
-use crate::native::watch::watch_config;
-
 #[napi]
 pub struct Watcher {
     pub origin: String,
@@ -42,26 +37,23 @@ impl Watcher {
         origin: String,
         additional_globs: Option<Vec<String>>,
         use_ignore: Option<bool>,
-    ) -> Result<Watcher> {
+    ) -> Watcher {
-        let watch_exec = Watchexec::new(InitConfig::default(), RuntimeConfig::default())
-            .map_err(anyhow::Error::from)?;
-
         // always have these globs come before the additional globs
         let mut globs = vec![".git/".into(), "node_modules/".into(), ".nx/".into()];
         if let Some(additional_globs) = additional_globs {
             globs.extend(additional_globs);
         }
 
-        Ok(Watcher {
+        Watcher {
             origin: if cfg!(window) {
                 origin.replace('/', "\\")
             } else {
                 origin
             },
-            watch_exec,
+            watch_exec: Arc::new(Watchexec::default()),
             additional_globs: globs,
             use_ignore: use_ignore.unwrap_or(true),
-        })
+        }
     }
 
     #[napi]
@@ -95,98 +87,76 @@ impl Watcher {
         callback_tsfn.unref(&env)?;
 
         let origin = self.origin.clone();
-        let watch_exec = self.watch_exec.clone();
+        self.watch_exec.config.on_action(move |mut action| {
-        let additional_globs = self.additional_globs.clone();
+            let signals: Vec<Signal> = action.signals().collect();
-        let use_ignore = self.use_ignore;
-        let start = async move {
-            let mut runtime = watch_config::create_runtime(
-                &origin,
-                &additional_globs
-                    .iter()
-                    .map(String::as_ref)
-                    .collect::<Vec<_>>(),
-                use_ignore,
-            )
-            .await?;
 
-            runtime.on_action(move |action: Action| {
+            if signals.contains(&Signal::Terminate) {
-                let ok_future = async { Ok::<(), Infallible>(()) };
+                trace!("terminate - ending watch");
-                let signals: Vec<Signal> = action.events.iter().flat_map(Event::signals).collect();
+                action.quit();
+                return action;
+            }
 
-                if signals.contains(&Signal::Terminate) {
+            if signals.contains(&Signal::Interrupt) {
-                    trace!("terminate - ending watch");
+                trace!("interrupt - ending watch");
-                    action.outcome(Outcome::both(Outcome::Stop, Outcome::Exit));
+                action.quit();
-                    return ok_future;
+                return action;
-                }
+            }
 
-                if signals.contains(&Signal::Interrupt) {
+            let mut origin_path = origin.clone();
-                    trace!("interrupt - ending watch");
+            if !origin_path.ends_with(MAIN_SEPARATOR) {
-                    action.outcome(Outcome::both(Outcome::Stop, Outcome::Exit));
+                origin_path.push(MAIN_SEPARATOR);
-                    return ok_future;
+            }
-                }
+            trace!(?origin_path);
 
-                let is_keyboard_eof = action
+            let events = action
-                    .events
+                .events
-                    .iter()
+                .par_iter()
-                    .any(|e| e.tags.contains(&Tag::Keyboard(Keyboard::Eof)));
+                .map(|ev| {
+                    let mut watch_event: WatchEventInternal = ev.into();
+                    watch_event.origin = Some(origin_path.clone());
+                    watch_event
+                })
+                .collect::<Vec<WatchEventInternal>>();
 
-                if is_keyboard_eof {
+            let mut group_events: HashMap<String, WatchEventInternal> = HashMap::new();
-                    trace!("ending watch");
+            for g in events.into_iter() {
-                    action.outcome(Outcome::both(Outcome::Stop, Outcome::Exit));
+                let path = g.path.display().to_string();
-                    return ok_future;
-                }
 
-                let mut origin_path = origin.clone();
+                // Delete > Create > Modify
-                if !origin_path.ends_with(MAIN_SEPARATOR) {
+                match group_events.entry(path) {
-                    origin_path.push(MAIN_SEPARATOR);
+                    // Delete should override anything
-                }
+                    Entry::Occupied(mut e) if matches!(g.r#type, EventType::delete) => {
-                trace!(?origin_path);
+                        e.insert(g);
-
+                    }
-                let events = action
+                    // Create should override update
-                    .events
+                    Entry::Occupied(mut e)
-                    .par_iter()
+                        if matches!(g.r#type, EventType::create)
-                    .map(|ev| {
+                            && matches!(e.get().r#type, EventType::update) =>
-                        let mut watch_event: WatchEventInternal = ev.into();
+                    {
-                        watch_event.origin = Some(origin_path.clone());
+                        e.insert(g);
-                        watch_event
+                    }
-                    })
+                    Entry::Occupied(_) => {}
-                    .collect::<Vec<WatchEventInternal>>();
+                    // If its empty, insert
-
+                    Entry::Vacant(e) => {
-                let mut group_events: HashMap<String, WatchEventInternal> = HashMap::new();
+                        e.insert(g);
-                for g in events.into_iter() {
-                    let path = g.path.display().to_string();
-
-                    // Delete > Create > Modify
-                    match group_events.entry(path) {
-                        // Delete should override anything
-                        Entry::Occupied(mut e) if matches!(g.r#type, EventType::delete) => {
-                            e.insert(g);
-                        }
-                        // Create should override update
-                        Entry::Occupied(mut e)
-                            if matches!(g.r#type, EventType::create)
-                                && matches!(e.get().r#type, EventType::update) =>
-                        {
-                            e.insert(g);
-                        }
-                        Entry::Occupied(_) => {}
-                        // If its empty, insert
-                        Entry::Vacant(e) => {
-                            e.insert(g);
-                        }
                     }
                 }
-                callback_tsfn.call(Ok(group_events), ThreadsafeFunctionCallMode::NonBlocking);
+            }
+            callback_tsfn.call(Ok(group_events), ThreadsafeFunctionCallMode::NonBlocking);
 
-                action.outcome(Outcome::Start);
+            action
-                ok_future
+        });
-            });
 
+        let origin = self.origin.clone();
+        let additional_globs = self.additional_globs.clone();
+        let use_ignore = self.use_ignore;
+        let watch_exec = self.watch_exec.clone();
+        let start = async move {
             trace!("configuring watch exec");
-            watch_exec
+            watch_exec.config.pathset([&origin.as_str()]);
-                .reconfigure(runtime)
+            watch_exec.config.filterer(
-                .map_err(anyhow::Error::from)?;
+                watch_filterer::create_filter(&origin, &additional_globs, use_ignore).await?,
-
+            );
             trace!("starting watch exec");
             watch_exec.main().await.map_err(anyhow::Error::from)?.ok();
             Ok(())