Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fs: add recursive watch for linux #45098

Merged
merged 38 commits into from
Oct 31, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
cdc9349
fs: add recursive watch to linux
anonrig Oct 19, 2022
296646b
fs: replace traverse with readdir for performance
anonrig Oct 21, 2022
77cb9a0
fs: fix linter issues
anonrig Oct 22, 2022
5b72449
fs: move linux watcher to internal/fs/watch
anonrig Oct 22, 2022
87866c3
test: add missing clear interval for linux
anonrig Oct 22, 2022
00ab12d
fs: add promise support to linux watcher
anonrig Oct 22, 2022
cb4b455
test: simplify start and update tests
anonrig Oct 22, 2022
4144949
fs: avoid prototype pollution
anonrig Oct 22, 2022
9a86410
test: improve fs.watch tests
anonrig Oct 23, 2022
8b36541
test: handle edge cases
anonrig Oct 23, 2022
89abb18
fs: handle more linux edge cases
anonrig Oct 23, 2022
f15251f
fs: update requested changes
anonrig Oct 23, 2022
cd240a6
fs: fix circular dependency
anonrig Oct 24, 2022
6311cf5
fs: improve tests
anonrig Oct 24, 2022
d8d2a0c
test: add url as parameter for fs.watch
anonrig Oct 24, 2022
632a4bc
test: update lint errors
anonrig Oct 25, 2022
df610ef
test: fix url error
anonrig Oct 25, 2022
8243dde
test: improve test-fs-watch-recursive.js
anonrig Oct 25, 2022
37a0839
fs: use arrays instead of sets
anonrig Oct 25, 2022
c7512ef
fs: remove lazy loading assert
anonrig Oct 25, 2022
6eee878
fs: revert certain changes
anonrig Oct 26, 2022
a6906f2
fs: add recursive validation to linux watcher
anonrig Oct 26, 2022
6061330
test: improve tests
anonrig Oct 26, 2022
9d596e8
fs: do not throw abort errors
anonrig Oct 26, 2022
d94f365
fs: rename recursive watch
anonrig Oct 26, 2022
b5161e5
fs: adjust implementation
anonrig Oct 26, 2022
6e4299d
fs: add ref and unref to fs.watch
anonrig Oct 26, 2022
2e5d4b3
fs: update async iterator implementation
anonrig Oct 27, 2022
cd3199b
fs: improve watcher
anonrig Oct 28, 2022
d69a565
fs: rename linux watcher
anonrig Oct 29, 2022
28ee387
fs: add support for symlinks
anonrig Oct 29, 2022
8e4e3dd
test: remove redundant test
anonrig Oct 29, 2022
e6019be
fs: rename to nonNativeWatcher
anonrig Oct 29, 2022
1e903a2
fs: update comments
anonrig Oct 30, 2022
b8b87f6
test: update symlink error message
anonrig Oct 30, 2022
ab4f1d2
fs: update implementation
anonrig Oct 30, 2022
c11bb62
test: make fs.watch errors more strict
anonrig Oct 31, 2022
bddb83f
fs: add documentation
anonrig Oct 31, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test: make fs.watch errors more strict
  • Loading branch information
anonrig committed Oct 31, 2022
commit c11bb628a293a5cb322939261e59707cc75e649d
65 changes: 28 additions & 37 deletions lib/internal/fs/recursive_watch.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@ const {
Promise,
PromisePrototypeThen,
SafeMap,
SafeSet,
StringPrototypeStartsWith,
StringPrototypeSlice,
StringPrototypeLastIndexOf,
SymbolAsyncIterator,
} = primordials;

Expand Down Expand Up @@ -39,7 +38,7 @@ function lazyLoadFsSync() {
return internalSync;
}

async function traverse(dir, files = new SafeMap()) {
async function traverse(dir, files = new SafeMap(), symbolicLinks = new SafeSet()) {
const { opendir } = lazyLoadFsPromises();

const filenames = await opendir(dir);
anonrig marked this conversation as resolved.
Show resolved Hide resolved
Expand All @@ -50,8 +49,9 @@ async function traverse(dir, files = new SafeMap()) {

files.set(f, file);

// Do not follow symbolic links
if (file.isSymbolicLink()) {
// Do not follow symbolic links
symbolicLinks.add(f);
} else if (file.isDirectory()) {
ArrayPrototypePush(subdirectories, traverse(f, files));
}
Expand All @@ -66,6 +66,7 @@ class FSWatcher extends EventEmitter {
#options = null;
#closed = false;
#files = new SafeMap();
#symbolicFiles = new SafeSet();
#rootPath = path.resolve();
#watchingFile = false;

Expand Down Expand Up @@ -112,32 +113,15 @@ class FSWatcher extends EventEmitter {
}

this.#files.clear();
this.#symbolicFiles.clear();
this.emit('close');
}

/**
* @param {string} file
* @return {string} the full path of `file` if we're watching a directory, or the
filename when watching a file.
*/
#getPath(file) {
// Always return the filename instead of the full path if watching a file (not a directory)
if (this.#watchingFile) {
return StringPrototypeSlice(this.#rootPath, StringPrototypeLastIndexOf(this.#rootPath, '/') + 1);
}

// When watching directories, and root directory is changed, return the full file path
if (file === this.#rootPath) {
return this.#rootPath;
}

// Otherwise return relative path for all files & folders
return path.relative(this.#rootPath, file);
}

#unwatchFiles(file) {
const { unwatchFile } = lazyLoadFsSync();

this.#symbolicFiles.delete(file);

for (const filename of this.#files.keys()) {
if (StringPrototypeStartsWith(filename, file)) {
unwatchFile(filename);
Expand All @@ -159,7 +143,11 @@ class FSWatcher extends EventEmitter {
const f = path.join(folder, file.name);

if (!this.#files.has(f)) {
this.emit('change', 'rename', this.#getPath(f));
this.emit('change', 'rename', path.relative(this.#rootPath, f));

if (file.isSymbolicLink()) {
this.#symbolicFiles.add(f);
}

if (file.isFile()) {
this.#watchFile(f);
Expand Down Expand Up @@ -187,25 +175,28 @@ class FSWatcher extends EventEmitter {

watchFile(file, {
persistent: this.#options.persistent,
}, (statWatcher, previousStatWatcher) => {
}, (currentStats, previousStats) => {
if (existingStat && !existingStat.isDirectory() &&
statWatcher.nlink !== 0 && existingStat.mtime?.getTime() === statWatcher.mtime?.getTime()) {
currentStats.nlink !== 0 && existingStat.mtimeMs === currentStats.mtimeMs) {
return;
}

this.#files.set(file, statWatcher);
this.#files.set(file, currentStats);

if (statWatcher.birthtimeMs === 0 && previousStatWatcher.birthtimeMs !== 0) {
if (currentStats.birthtimeMs === 0 && previousStats.birthtimeMs !== 0) {
// The file is now deleted
this.#files.delete(file);
this.emit('change', 'rename', this.#getPath(file));
this.emit('change', 'rename', path.relative(this.#rootPath, file));
this.#unwatchFiles(file);
} else {
this.emit('change', 'change', this.#getPath(file));

if (statWatcher.isDirectory() && !statWatcher.isSymbolicLink()) {
this.#watchFolder(file);
}
} else if (file === this.#rootPath && this.#watchingFile) {
// This case will only be triggered when watching a file with fs.watch
this.emit('change', 'change', path.basename(file));
} else if (this.#symbolicFiles.has(file)) {
// Stats from watchFile does not return correct value for currentStats.isSymbolicLink()
// Since it is only valid when using fs.lstat(). Therefore, check the existing symbolic files.
this.emit('change', 'rename', path.relative(this.#rootPath, file));
} else if (currentStats.isDirectory()) {
this.#watchFolder(file);
}
});
}
Expand All @@ -224,7 +215,7 @@ class FSWatcher extends EventEmitter {
this.#files.set(filename, file);

PromisePrototypeThen(
traverse(filename, this.#files),
traverse(filename, this.#files, this.#symbolicFiles),
() => {
for (const f of this.#files.keys()) {
this.#watchFile(f);
Expand Down
60 changes: 36 additions & 24 deletions test/parallel/test-fs-watch-recursive-symlink.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,25 @@ tmpdir.refresh();
(async () => {
// Add a recursive symlink to the parent folder

const testsubdir = fs.mkdtempSync(testDir + path.sep);
const file = 'file-1.txt';
const filePath = path.join(testsubdir, file);
const watcher = fs.watch(testsubdir, { recursive: true });
const testDirectory = fs.mkdtempSync(testDir + path.sep);

const symlinkFile = path.join(testsubdir, 'symlink-folder');
fs.symlinkSync(testsubdir, symlinkFile);
// Do not use `testDirectory` as base. It will hang the tests.
const rootDirectory = path.join(testDirectory, 'test-1');
fs.mkdirSync(rootDirectory);

const filePath = path.join(rootDirectory, 'file.txt');

const symlinkFolder = path.join(rootDirectory, 'symlink-folder');
fs.symlinkSync(rootDirectory, symlinkFolder);


const watcher = fs.watch(rootDirectory, { recursive: true });
let watcherClosed = false;
watcher.on('change', function(event, filename) {
assert.ok(event === 'change' || event === 'rename');
assert.ok(event === 'rename', `Received ${event}`);
assert.ok(filename === path.basename(symlinkFolder) || filename === path.basename(filePath), `Received ${filename}`);

if (filename === file) {
if (filename === path.basename(filePath)) {
watcher.close();
watcherClosed = true;
}
Expand All @@ -51,36 +57,42 @@ tmpdir.refresh();
})().then(common.mustCall());

(async () => {
// Add a recursive symlink outside the parent folder
// This test checks how a symlink to outside the tracking folder can trigger change
// tmp/sub-directory/tracking-folder/symlink-folder -> tmp/sub-directory

const testsubdir = fs.mkdtempSync(testDir + path.sep);
const file = 'file-1.txt';
const filePath = path.join(testsubdir, file);
const rootDirectory = fs.mkdtempSync(testDir + path.sep);

const trackFolder = path.join(testsubdir, 'tracking-folder');
const acceptableFile = 'acceptable-1.txt';
const acceptableFilePath = path.join(trackFolder, acceptableFile);
fs.mkdirSync(trackFolder);
const subDirectory = path.join(rootDirectory, 'sub-directory');
fs.mkdirSync(subDirectory);

const symlinkFile = path.join(trackFolder, 'symlink-folder');
fs.symlinkSync(testsubdir, symlinkFile);
const trackingSubDirectory = path.join(subDirectory, 'tracking-folder');
fs.mkdirSync(trackingSubDirectory);

const watcher = fs.watch(trackFolder, { recursive: true });
const symlinkFolder = path.join(trackingSubDirectory, 'symlink-folder');
fs.symlinkSync(subDirectory, symlinkFolder);

const forbiddenFile = path.join(subDirectory, 'forbidden.txt');
const acceptableFile = path.join(trackingSubDirectory, 'acceptable.txt');

const watcher = fs.watch(trackingSubDirectory, { recursive: true });
let watcherClosed = false;
watcher.on('change', function(event, filename) {
assert.ok(event === 'change' || event === 'rename');
assert.ok(filename === 'symlink-folder' || filename === trackFolder || filename === acceptableFile, `Received filename ${filename}`);
// macOS will only change the following events:
// { event: 'rename', filename: 'symlink-folder' }
// { event: 'rename', filename: 'acceptable.txt' }
assert.ok(event === 'rename', `Received ${event}`);
assert.ok(filename === path.basename(symlinkFolder) || filename === path.basename(acceptableFile), `Received ${filename}`);

if (filename === acceptableFile) {
if (filename === path.basename(acceptableFile)) {
watcher.close();
watcherClosed = true;
}
});

await setTimeout(common.platformTimeout(100));
fs.writeFileSync(filePath, 'world');
fs.writeFileSync(forbiddenFile, 'world');
await setTimeout(common.platformTimeout(100));
fs.writeFileSync(acceptableFilePath, 'acceptable');
fs.writeFileSync(acceptableFile, 'acceptable');

process.once('exit', function() {
assert(watcherClosed, 'watcher Object was not closed');
Expand Down
Loading