miel.truyen/nx
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

fix(core): prioritize nxignore for watcher updates (#20975)

Browse Source
This commit is contained in:
Jonathan Cammisuli 2024-01-04 07:41:20 -05:00 committed by GitHub
parent e31c179c02
commit a2f7ae7f22
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 541 additions and 453 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

647
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

11
packages/nx/Cargo.toml
View File

@ -8,11 +8,12 @@ anyhow = "1.0.71"
colored = "2"
crossbeam-channel = '0.5'
dashmap = { version = "5.5.3", features = ["rayon"] }
dunce = "1"
fs_extra = "1.3.0"
globset = "0.4.10"
hashbrown = { version = "0.14.3", features = ["rayon", "rkyv"] }
ignore = '0.4'
ignore-files = "1.3.0"
ignore-files = "2.0.0"
itertools = "0.10.5"
once_cell = "1.18.0"
parking_lot = { version = "0.12.1", features = ["send_guard"] }
@ -31,10 +32,10 @@ tokio = { version = "1.28.2", features = ["fs"] }
tracing = "0.1.37"
tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
walkdir = '2.3.3'
watchexec = "2.3.0"
watchexec-events = "1.0.0"
watchexec-filterer-ignore = "1.2.1"
watchexec-signals = "1.0.0"
watchexec = "3.0.1"
watchexec-events = "2.0.1"
watchexec-filterer-ignore = "3.0.0"
watchexec-signals = "2.1.0"
xxhash-rust = { version = '0.8.5', features = ['xxh3', 'xxh64'] }
swc_common = "0.31.16"
swc_ecma_parser = { version = "0.137.1", features = ["typescript"] }

27
packages/nx/src/native/tests/watcher.spec.ts
View File

@ -8,11 +8,14 @@ describe('watcher', () => {
temp = new TempFs('watch-dir');
temp.createFilesSync({
'.gitignore': 'node_modules/\n.env.local',
'.nxignore': 'app2/\n!.env.local',
'.nxignore': 'app2/\n!.env.*\nboo.txt',
'.env.local': '',
'app1/main.js': '',
'app1/main.css': '',
'app2/main.js': '',
'inner/.gitignore': '.env.inner',
'inner/boo.txt': '',
'inner/.env.inner': '',
'nested-ignore/.gitignore': '*',
'nested-ignore/file.js': '',
'node_modules/module/index.js': '',
@ -51,7 +54,7 @@ describe('watcher', () => {
await wait();
temp.createFileSync('app1/main.html', JSON.stringify({}));
});
}, 10000);
}, 15000);
it('should trigger the callback when files are updated', async () => {
return new Promise<void>(async (done) => {
@ -76,7 +79,7 @@ describe('watcher', () => {
await wait();
temp.appendFile('app1/main.js', 'update');
});
}, 10000);
}, 15000);
it('should watch file renames', async () => {
return new Promise<void>(async (done) => {
@ -103,7 +106,7 @@ describe('watcher', () => {
await wait();
temp.renameFile('app1/main.js', 'app1/rename.js');
});
}, 10000);
}, 15000);
it('should trigger on deletes', async () => {
return new Promise<void>(async (done) => {
@ -125,7 +128,7 @@ describe('watcher', () => {
await wait();
temp.removeFileSync('app1/main.js');
});
}, 10000);
}, 15000);
it('should ignore nested gitignores', async () => {
return new Promise<void>(async (done) => {
@ -137,7 +140,7 @@ describe('watcher', () => {
expect(paths).toMatchInlineSnapshot(`
[
{
"path": "boo.txt",
"path": "bar.txt",
"type": "create",
},
]
@ -149,21 +152,27 @@ describe('watcher', () => {
// should not be triggered
temp.createFileSync('nested-ignore/hello1.txt', '');
await wait();
temp.createFileSync('boo.txt', '');
temp.createFileSync('bar.txt', '');
});
}, 10000);
}, 15000);
it('should include files that are negated in nxignore but are ignored in gitignore', async () => {
it('prioritize nxignore over gitignores', async () => {
return new Promise<void>(async (done) => {
await wait();
watcher = new Watcher(temp.tempDir);
watcher.watch((err, paths) => {
expect(paths.some(({ path }) => path === '.env.local')).toBeTruthy();
expect(
paths.some(({ path }) => path === 'inner/.env.inner')
).toBeTruthy();
expect(paths.some(({ path }) => path === 'inner/boo.txt')).toBeFalsy();
done();
});
await wait(2000);
temp.appendFile('.env.local', 'hello');
temp.appendFile('inner/.env.inner', 'hello');
temp.appendFile('inner/boo.txt', 'hello');
});
}, 15000);
});

1
packages/nx/src/native/watch/mod.rs
View File

@ -1,5 +1,4 @@
mod types;
mod utils;
mod watch_config;
mod watch_filterer;
mod watcher;

54
packages/nx/src/native/watch/watch_config.rs
View File

@ -1,54 +0,0 @@
use std::sync::Arc;
use ignore_files::IgnoreFilter;
use tracing::trace;
use watchexec::config::RuntimeConfig;
use watchexec_filterer_ignore::IgnoreFilterer;
use crate::native::watch::utils::{get_ignore_files, get_nx_ignore};
use crate::native::watch::watch_filterer::WatchFilterer;
pub(super) async fn create_runtime(
origin: &str,
additional_globs: &[&str],
use_ignore: bool,
) -> napi::Result<RuntimeConfig> {
let ignore_files = get_ignore_files(use_ignore, origin);
let nx_ignore_file = get_nx_ignore(origin);
trace!(
?use_ignore,
?additional_globs,
?ignore_files,
"Using these ignore files for the watcher"
);
let mut filter = if let Some(ignore_files) = ignore_files {
IgnoreFilter::new(origin, &ignore_files)
.await
.map_err(anyhow::Error::from)?
} else {
IgnoreFilter::empty(origin)
};
filter
.add_globs(additional_globs, Some(&origin.into()))
.map_err(anyhow::Error::from)?;
// always add the .nxignore file after all other ignores are loaded so that it has the highest priority
if let Some(nx_ignore_file) = nx_ignore_file {
filter
.add_file(&nx_ignore_file)
.await
.map_err(anyhow::Error::from)?;
}
let mut runtime = RuntimeConfig::default();
runtime.filterer(Arc::new(WatchFilterer {
inner: IgnoreFilterer(filter),
}));
// let watch_directories = get_watch_directories(origin);
// trace!(directories = ?watch_directories, "watching");
runtime.pathset([&origin]);
Ok(runtime)
}

94
packages/nx/src/native/watch/watch_filterer.rs
View File

@ -1,16 +1,53 @@
use ignore::Match;
use tracing::trace;
use watchexec::error::RuntimeError;
use watchexec::filter::Filterer;
use watchexec_events::filekind::{CreateKind, FileEventKind, ModifyKind, RemoveKind};
use ignore_files::IgnoreFilter;
use watchexec_events::{Event, FileType, Priority, Source, Tag};
use watchexec_filterer_ignore::IgnoreFilterer;
use crate::native::watch::utils::transform_event;
use crate::native::watch::utils::{get_ignore_files, get_nx_ignore, transform_event};
#[derive(Debug)]
pub struct WatchFilterer {
pub inner: IgnoreFilterer,
pub nx_ignore: Option<IgnoreFilter>,
pub git_ignore: IgnoreFilterer,
}
impl WatchFilterer {
fn filter_event(&self, event: &Event, priority: Priority) -> bool {
let mut pass = true;
for (path, file_type) in event.paths() {
let path = dunce::simplified(path);
let is_dir = file_type.map_or(false, |t| matches!(t, FileType::Dir));
let nx_ignore_match_type = if let Some(nx_ignore) = &self.nx_ignore {
nx_ignore.match_path(path, is_dir)
} else {
Match::None
};
// if the nxignore file contains this file as a whitelist,
// we do not want gitignore to filter it out, so it will always pass as true
if matches!(nx_ignore_match_type, Match::Whitelist(_)) {
trace!(?path, "nxignore whitelist match, ignoring gitignore");
pass &= true;
// If the nxignore file contains this file as an ignore,
// then there's no point in checking the gitignore file
} else if matches!(nx_ignore_match_type, Match::Ignore(_)) {
trace!(?path, "nxignore ignore match, ignoring gitignore");
pass &= false;
} else {
pass &= self
.git_ignore
.check_event(event, priority)
.expect("git ignore check never errors")
}
}
pass
}
}
/// Used to filter out events that that come from watchexec
@ -19,12 +56,11 @@ impl Filterer for WatchFilterer {
let transformed = transform_event(watch_event);
let event = transformed.as_ref().unwrap_or(watch_event);
if !self.inner.check_event(event, priority)? {
trace!(?event, "checking if event is valid");
if !self.filter_event(event, priority) {
return Ok(false);
}
trace!(?event, "checking if event is valid");
//
// Tags will be a Vec that contains multiple types of information for a given event
// We are only interested if:
@ -67,3 +103,51 @@ impl Filterer for WatchFilterer {
Ok(true)
}
}
pub(super) async fn create_filter(
origin: &str,
additional_globs: &[String],
use_ignore: bool,
) -> anyhow::Result<WatchFilterer> {
let ignore_files = get_ignore_files(use_ignore, origin);
let nx_ignore_file = get_nx_ignore(origin);
trace!(
?use_ignore,
?additional_globs,
?ignore_files,
"Using these ignore files for the watcher"
);
let mut git_ignore = if let Some(ignore_files) = ignore_files {
IgnoreFilter::new(origin, &ignore_files)
.await
.map_err(anyhow::Error::from)?
} else {
IgnoreFilter::empty(origin)
};
git_ignore
.add_globs(
&additional_globs
.iter()
.map(String::as_ref)
.collect::<Vec<_>>(),
Some(&origin.into()),
)
.map_err(anyhow::Error::from)?;
let nx_ignore = if let Some(nx_ignore_file) = nx_ignore_file {
Some(
IgnoreFilter::new(origin, &[nx_ignore_file])
.await
.map_err(anyhow::Error::from)?,
)
} else {
None
};
Ok(WatchFilterer {
git_ignore: IgnoreFilterer(git_ignore),
nx_ignore,
})
}

160
packages/nx/src/native/watch/watcher.rs
View File

@ -1,10 +1,10 @@
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::convert::Infallible;
use std::path::MAIN_SEPARATOR;
use std::sync::Arc;
use crate::native::watch::types::{EventType, WatchEvent, WatchEventInternal};
use crate::native::watch::watch_filterer;
use napi::bindgen_prelude::*;
use napi::threadsafe_function::{
ThreadSafeCallContext, ThreadsafeFunction, ThreadsafeFunctionCallMode,
@ -13,15 +13,10 @@ use napi::{Env, JsFunction, JsObject};
use rayon::prelude::*;
use tracing::trace;
use tracing_subscriber::EnvFilter;
use watchexec::action::{Action, Outcome};
use watchexec::config::{InitConfig, RuntimeConfig};
use watchexec::event::Tag;
use watchexec::Watchexec;
use watchexec_events::{Event, Keyboard, Priority};
use watchexec_events::{Event, Priority, Tag};
use watchexec_signals::Signal;
use crate::native::watch::watch_config;
#[napi]
pub struct Watcher {
pub origin: String,
@ -42,26 +37,23 @@ impl Watcher {
origin: String,
additional_globs: Option<Vec<String>>,
use_ignore: Option<bool>,
) -> Result<Watcher> {
let watch_exec = Watchexec::new(InitConfig::default(), RuntimeConfig::default())
.map_err(anyhow::Error::from)?;
) -> Watcher {
// always have these globs come before the additional globs
let mut globs = vec![".git/".into(), "node_modules/".into(), ".nx/".into()];
if let Some(additional_globs) = additional_globs {
globs.extend(additional_globs);
}
Ok(Watcher {
Watcher {
origin: if cfg!(window) {
origin.replace('/', "\\")
} else {
origin
},
watch_exec,
watch_exec: Arc::new(Watchexec::default()),
additional_globs: globs,
use_ignore: use_ignore.unwrap_or(true),
})
}
}
#[napi]
@ -95,98 +87,76 @@ impl Watcher {
callback_tsfn.unref(&env)?;
let origin = self.origin.clone();
let watch_exec = self.watch_exec.clone();
let additional_globs = self.additional_globs.clone();
let use_ignore = self.use_ignore;
let start = async move {
let mut runtime = watch_config::create_runtime(
&origin,
&additional_globs
.iter()
.map(String::as_ref)
.collect::<Vec<_>>(),
use_ignore,
)
.await?;
self.watch_exec.config.on_action(move |mut action| {
let signals: Vec<Signal> = action.signals().collect();
runtime.on_action(move |action: Action| {
let ok_future = async { Ok::<(), Infallible>(()) };
let signals: Vec<Signal> = action.events.iter().flat_map(Event::signals).collect();
if signals.contains(&Signal::Terminate) {
trace!("terminate - ending watch");
action.quit();
return action;
}
if signals.contains(&Signal::Terminate) {
trace!("terminate - ending watch");
action.outcome(Outcome::both(Outcome::Stop, Outcome::Exit));
return ok_future;
}
if signals.contains(&Signal::Interrupt) {
trace!("interrupt - ending watch");
action.quit();
return action;
}
if signals.contains(&Signal::Interrupt) {
trace!("interrupt - ending watch");
action.outcome(Outcome::both(Outcome::Stop, Outcome::Exit));
return ok_future;
}
let mut origin_path = origin.clone();
if !origin_path.ends_with(MAIN_SEPARATOR) {
origin_path.push(MAIN_SEPARATOR);
}
trace!(?origin_path);
let is_keyboard_eof = action
.events
.iter()
.any(|e| e.tags.contains(&Tag::Keyboard(Keyboard::Eof)));
let events = action
.events
.par_iter()
.map(|ev| {
let mut watch_event: WatchEventInternal = ev.into();
watch_event.origin = Some(origin_path.clone());
watch_event
})
.collect::<Vec<WatchEventInternal>>();
if is_keyboard_eof {
trace!("ending watch");
action.outcome(Outcome::both(Outcome::Stop, Outcome::Exit));
return ok_future;
}
let mut group_events: HashMap<String, WatchEventInternal> = HashMap::new();
for g in events.into_iter() {
let path = g.path.display().to_string();
let mut origin_path = origin.clone();
if !origin_path.ends_with(MAIN_SEPARATOR) {
origin_path.push(MAIN_SEPARATOR);
}
trace!(?origin_path);
let events = action
.events
.par_iter()
.map(|ev| {
let mut watch_event: WatchEventInternal = ev.into();
watch_event.origin = Some(origin_path.clone());
watch_event
})
.collect::<Vec<WatchEventInternal>>();
let mut group_events: HashMap<String, WatchEventInternal> = HashMap::new();
for g in events.into_iter() {
let path = g.path.display().to_string();
// Delete > Create > Modify
match group_events.entry(path) {
// Delete should override anything
Entry::Occupied(mut e) if matches!(g.r#type, EventType::delete) => {
e.insert(g);
}
// Create should override update
Entry::Occupied(mut e)
if matches!(g.r#type, EventType::create)
&& matches!(e.get().r#type, EventType::update) =>
{
e.insert(g);
}
Entry::Occupied(_) => {}
// If its empty, insert
Entry::Vacant(e) => {
e.insert(g);
}
// Delete > Create > Modify
match group_events.entry(path) {
// Delete should override anything
Entry::Occupied(mut e) if matches!(g.r#type, EventType::delete) => {
e.insert(g);
}
// Create should override update
Entry::Occupied(mut e)
if matches!(g.r#type, EventType::create)
&& matches!(e.get().r#type, EventType::update) =>
{
e.insert(g);
}
Entry::Occupied(_) => {}
// If its empty, insert
Entry::Vacant(e) => {
e.insert(g);
}
}
callback_tsfn.call(Ok(group_events), ThreadsafeFunctionCallMode::NonBlocking);
}
callback_tsfn.call(Ok(group_events), ThreadsafeFunctionCallMode::NonBlocking);
action.outcome(Outcome::Start);
ok_future
});
action
});
let origin = self.origin.clone();
let additional_globs = self.additional_globs.clone();
let use_ignore = self.use_ignore;
let watch_exec = self.watch_exec.clone();
let start = async move {
trace!("configuring watch exec");
watch_exec
.reconfigure(runtime)
.map_err(anyhow::Error::from)?;
watch_exec.config.pathset([&origin.as_str()]);
watch_exec.config.filterer(
watch_filterer::create_filter(&origin, &additional_globs, use_ignore).await?,
);
trace!("starting watch exec");
watch_exec.main().await.map_err(anyhow::Error::from)?.ok();
Ok(())