feat(core): introduce workspace file archive (#20471)

This commit is contained in:
Jonathan Cammisuli 2023-11-30 10:54:28 -05:00 committed by GitHub
parent ff9d95568f
commit 15c2181664
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 594 additions and 130 deletions

274
Cargo.lock generated
View File

@ -14,14 +14,26 @@ dependencies = [
[[package]]
name = "ahash"
version = "0.8.3"
version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd"
dependencies = [
"getrandom",
"once_cell",
"version_check",
]
[[package]]
name = "ahash"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
dependencies = [
"cfg-if",
"getrandom",
"once_cell",
"version_check",
"zerocopy",
]
[[package]]
@ -78,7 +90,7 @@ dependencies = [
"proc-macro2",
"quote",
"swc_macros_common",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -98,7 +110,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -109,7 +121,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -156,6 +168,18 @@ version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
[[package]]
name = "bitvec"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"
dependencies = [
"funty",
"radium",
"tap",
"wyz",
]
[[package]]
name = "bstr"
version = "1.4.0"
@ -177,6 +201,28 @@ dependencies = [
"num-traits",
]
[[package]]
name = "bytecheck"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627"
dependencies = [
"bytecheck_derive",
"ptr_meta",
"simdutf8",
]
[[package]]
name = "bytecheck_derive"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.107",
]
[[package]]
name = "bytes"
version = "1.4.0"
@ -290,7 +336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd4056f63fce3b82d852c3da92b08ea59959890813a7f4ce9c0ff85b10cf301b"
dependencies = [
"quote",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -300,7 +346,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
dependencies = [
"cfg-if",
"hashbrown",
"hashbrown 0.14.3",
"lock_api",
"once_cell",
"parking_lot_core",
@ -429,7 +475,7 @@ dependencies = [
"pmutil",
"proc-macro2",
"swc_macros_common",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -447,6 +493,12 @@ dependencies = [
"libc",
]
[[package]]
name = "funty"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
[[package]]
name = "futures"
version = "0.3.29"
@ -503,7 +555,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -783,13 +835,23 @@ dependencies = [
[[package]]
name = "hashbrown"
version = "0.14.0"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
"ahash",
"ahash 0.7.7",
]
[[package]]
name = "hashbrown"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
dependencies = [
"ahash 0.8.6",
"allocator-api2",
"rayon",
"rkyv",
]
[[package]]
@ -926,7 +988,7 @@ dependencies = [
"pmutil",
"proc-macro2",
"quote",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -944,12 +1006,6 @@ version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
[[package]]
name = "json_comments"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41ee439ee368ba4a77ac70d04f14015415af8600d6c894dc1f11bd79758c57d5"
[[package]]
name = "kqueue"
version = "1.0.7"
@ -1152,7 +1208,7 @@ checksum = "4c65c625186a9bcce6699394bee511e1b1aec689aa7e3be1bf4e996e75834153"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -1366,7 +1422,7 @@ dependencies = [
"dashmap",
"fs_extra",
"globset",
"hashbrown",
"hashbrown 0.14.3",
"ignore",
"ignore-files",
"itertools",
@ -1378,6 +1434,7 @@ dependencies = [
"parking_lot",
"rayon",
"regex",
"rkyv",
"swc_common",
"swc_ecma_ast",
"swc_ecma_dep_graph",
@ -1387,7 +1444,6 @@ dependencies = [
"tokio",
"tracing",
"tracing-subscriber",
"tsconfig",
"walkdir",
"watchexec",
"watchexec-events",
@ -1514,7 +1570,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -1586,14 +1642,40 @@ dependencies = [
]
[[package]]
name = "quote"
version = "1.0.27"
name = "ptr_meta"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500"
checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1"
dependencies = [
"ptr_meta_derive",
]
[[package]]
name = "ptr_meta_derive"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.107",
]
[[package]]
name = "quote"
version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
dependencies = [
"proc-macro2",
]
[[package]]
name = "radium"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
[[package]]
name = "radix_trie"
version = "0.2.1"
@ -1738,6 +1820,43 @@ version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
[[package]]
name = "rend"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2571463863a6bd50c32f94402933f03457a3fbaf697a707c5be741e459f08fd"
dependencies = [
"bytecheck",
]
[[package]]
name = "rkyv"
version = "0.7.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58"
dependencies = [
"bitvec",
"bytecheck",
"hashbrown 0.12.3",
"ptr_meta",
"rend",
"rkyv_derive",
"seahash",
"tinyvec",
"uuid",
]
[[package]]
name = "rkyv_derive"
version = "0.7.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.107",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
@ -1758,12 +1877,6 @@ dependencies = [
"windows-sys 0.45.0",
]
[[package]]
name = "ryu"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
[[package]]
name = "same-file"
version = "1.0.6"
@ -1785,6 +1898,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "seahash"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
[[package]]
name = "semver"
version = "1.0.17"
@ -1811,17 +1930,6 @@ dependencies = [
"syn 1.0.107",
]
[[package]]
name = "serde_json"
version = "1.0.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "sha1_smol"
version = "1.0.0"
@ -1846,6 +1954,12 @@ dependencies = [
"libc",
]
[[package]]
name = "simdutf8"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
[[package]]
name = "siphasher"
version = "0.3.10"
@ -1939,7 +2053,7 @@ dependencies = [
"proc-macro2",
"quote",
"swc_macros_common",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -1962,7 +2076,7 @@ version = "0.31.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6414bd4e553f5638961d39b07075ffd37a3d63176829592f4a5900260d94ca1"
dependencies = [
"ahash",
"ahash 0.8.6",
"ast_node",
"better_scoped_tls",
"cfg-if",
@ -2054,7 +2168,7 @@ dependencies = [
"pmutil",
"proc-macro2",
"quote",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -2066,7 +2180,7 @@ dependencies = [
"pmutil",
"proc-macro2",
"quote",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -2090,7 +2204,7 @@ dependencies = [
"proc-macro2",
"quote",
"swc_macros_common",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -2106,15 +2220,21 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.15"
version = "2.0.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822"
checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "tap"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
[[package]]
name = "tempfile"
version = "3.5.0"
@ -2164,7 +2284,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -2245,7 +2365,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -2280,7 +2400,7 @@ checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.15",
"syn 2.0.39",
]
[[package]]
@ -2332,19 +2452,6 @@ dependencies = [
"stable_deref_trait",
]
[[package]]
name = "tsconfig"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c89ed286b13fd7e343eb628d8511fb4fdc99053acccb2263897e0d89526462b"
dependencies = [
"json_comments",
"regex",
"serde",
"serde_json",
"thiserror",
]
[[package]]
name = "typed-arena"
version = "2.0.2"
@ -2407,6 +2514,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "uuid"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560"
[[package]]
name = "valuable"
version = "0.1.0"
@ -2695,8 +2808,37 @@ version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
[[package]]
name = "wyz"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed"
dependencies = [
"tap",
]
[[package]]
name = "xxhash-rust"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70"
[[package]]
name = "zerocopy"
version = "0.7.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.7.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.39",
]

View File

@ -10,7 +10,7 @@ crossbeam-channel = '0.5'
dashmap = { version = "5.5.3", features = ["rayon"] }
fs_extra = "1.3.0"
globset = "0.4.10"
hashbrown = { version = "0.14.0", features = ["rayon"] }
hashbrown = { version = "0.14.3", features = ["rayon", "rkyv"] }
ignore = '0.4'
ignore-files = "1.3.0"
itertools = "0.10.5"
@ -25,11 +25,11 @@ napi-derive = '2.9.3'
nom = '7.1.3'
regex = "1.9.1"
rayon = "1.7.0"
rkyv = { version = "0.7", features = ["validation"] }
thiserror = "1.0.40"
tokio = { version = "1.28.2", features = ["fs"] }
tracing = "0.1.37"
tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
tsconfig = "0.2"
walkdir = '2.3.3'
watchexec = "2.3.0"
watchexec-events = "1.0.0"

View File

@ -169,7 +169,7 @@ export class Watcher {
}
export class WorkspaceContext {
workspaceRoot: string
constructor(workspaceRoot: string)
constructor(workspaceRoot: string, cacheDir: string)
getWorkspaceFiles(projectRootMap: Record<string, string>): NxWorkspaceFiles
glob(globs: Array<string>, exclude?: Array<string> | undefined | null): Array<string>
hashFilesMatchingGlob(globs: Array<string>, exclude?: Array<string> | undefined | null): string

View File

@ -666,7 +666,6 @@ fn find_imports(
mod find_imports {
use super::*;
use crate::native::glob::build_glob_set;
use crate::native::utils::Normalize;
use crate::native::walker::nx_walker;
use assert_fs::prelude::*;
use assert_fs::TempDir;
@ -1363,8 +1362,8 @@ import('./dynamic-import.vue')
let glob = build_glob_set(&["**/*.[jt]s"]).unwrap();
let files = nx_walker(root.clone())
.filter(|(full_path, _)| glob.is_match(full_path))
.map(|(full_path, _)| full_path.to_normalized_string())
.filter(|file| glob.is_match(&file.full_path))
.map(|file| file.full_path)
.collect::<Vec<_>>();
let results: HashMap<_, _> =

View File

@ -3,6 +3,7 @@ import { TempFs } from '../../internal-testing-utils/temp-fs';
import { NxJsonConfiguration } from '../../config/nx-json';
import { dirname, join } from 'path';
import { readJsonFile } from '../../utils/fileutils';
import { cacheDirectoryForWorkspace } from '../../utils/cache-directory';
describe('workspace files', () => {
function createParseConfigurationsFunction(tempDir: string) {
@ -49,16 +50,17 @@ describe('workspace files', () => {
'./nested/non-project/file.txt': '',
});
const context = new WorkspaceContext(fs.tempDir);
let { projectFileMap, globalFiles } = await context.getWorkspaceFiles(
{
'libs/project1': 'project1',
'libs/project2': 'project2',
'libs/project3': 'project3',
'libs/nested/project': 'nested-project',
'libs/package-project': 'package-project'
}
const context = new WorkspaceContext(
fs.tempDir,
cacheDirectoryForWorkspace(fs.tempDir)
);
let { projectFileMap, globalFiles } = await context.getWorkspaceFiles({
'libs/project1': 'project1',
'libs/project2': 'project2',
'libs/project3': 'project3',
'libs/nested/project': 'nested-project',
'libs/package-project': 'package-project',
});
expect(projectFileMap).toMatchInlineSnapshot(`
{
@ -149,14 +151,15 @@ describe('workspace files', () => {
'./jest.config.js': '',
});
const context = new WorkspaceContext(fs.tempDir);
const { globalFiles, projectFileMap } = await context.getWorkspaceFiles(
{
'.': 'repo-name'
}
const context = new WorkspaceContext(
fs.tempDir,
cacheDirectoryForWorkspace(fs.tempDir)
);
const { globalFiles, projectFileMap } = await context.getWorkspaceFiles({
'.': 'repo-name',
});
expect(globalFiles).toEqual([]);
expect(projectFileMap['repo-name']).toMatchInlineSnapshot(`
[

View File

@ -0,0 +1,19 @@
use std::fs::Metadata;
#[cfg(target_os = "macos")]
pub fn get_mod_time(metadata: &Metadata) -> i64 {
use std::os::macos::fs::MetadataExt;
metadata.st_mtime()
}
#[cfg(target_os = "windows")]
pub fn get_mod_time(metadata: &Metadata) -> i64 {
use std::os::windows::fs::MetadataExt;
metadata.last_write_time() as i64
}
#[cfg(target_os = "linux")]
pub fn get_mod_time(metadata: &Metadata) -> i64 {
use std::os::unix::fs::MetadataExt;
metadata.mtime()
}

View File

@ -1,6 +1,8 @@
mod find_matching_projects;
mod get_mod_time;
mod normalize_trait;
pub mod path;
pub use find_matching_projects::*;
pub use get_mod_time::*;
pub use normalize_trait::Normalize;

View File

@ -3,17 +3,17 @@ use std::path::{Path, PathBuf};
impl Normalize for Path {
fn to_normalized_string(&self) -> String {
normalize_path(self)
normalize_nx_path(self)
}
}
impl Normalize for PathBuf {
fn to_normalized_string(&self) -> String {
normalize_path(self)
normalize_nx_path(self)
}
}
fn normalize_path<P>(path: P) -> String
fn normalize_nx_path<P>(path: P) -> String
where
P: AsRef<Path>,
{

View File

@ -8,8 +8,16 @@ use tracing::trace;
use crate::native::glob::build_glob_set;
use crate::native::utils::{get_mod_time, Normalize};
use walkdir::WalkDir;
#[derive(PartialEq, Debug, Ord, PartialOrd, Eq, Clone)]
pub struct NxFile {
pub full_path: String,
pub normalized_path: String,
pub mod_time: i64,
}
/// Walks the directory in a single thread and does not ignore any files
/// Should only be used for small directories, and not traversing the whole workspace
pub fn nx_walker_sync<'a, P>(directory: P) -> impl Iterator<Item = PathBuf>
@ -36,7 +44,7 @@ where
}
/// Walk the directory and ignore files from .gitignore and .nxignore
pub fn nx_walker<P>(directory: P) -> impl Iterator<Item = (PathBuf, PathBuf)>
pub fn nx_walker<P>(directory: P) -> impl Iterator<Item = NxFile>
where
P: AsRef<Path>,
{
@ -80,8 +88,16 @@ where
return Continue;
};
tx.send((dir_entry.path().to_owned(), file_path.to_owned()))
.ok();
let Ok(metadata) = dir_entry.metadata() else {
return Continue;
};
tx.send(NxFile {
full_path: String::from(dir_entry.path().to_string_lossy()),
normalized_path: file_path.to_normalized_string(),
mod_time: get_mod_time(&metadata),
})
.ok();
Continue
})
@ -100,8 +116,6 @@ mod test {
use assert_fs::prelude::*;
use assert_fs::TempDir;
use crate::native::utils::Normalize;
use super::*;
///
@ -133,6 +147,10 @@ mod test {
let mut content = nx_walker(&temp_dir).collect::<Vec<_>>();
content.sort();
let content = content
.into_iter()
.map(|f| (f.full_path.into(), f.normalized_path.into()))
.collect::<Vec<_>>();
assert_eq!(
content,
vec![
@ -173,7 +191,12 @@ nested/child-two/
let mut file_names = nx_walker(temp_dir)
.into_iter()
.map(|(_, p)| p.to_normalized_string())
.map(
|NxFile {
normalized_path: relative_path,
..
}| relative_path,
)
.collect::<Vec<_>>();
file_names.sort();

View File

@ -1,14 +1,13 @@
use napi::bindgen_prelude::External;
use std::collections::HashMap;
use crate::native::hasher::{hash, hash_file_path};
use crate::native::hasher::hash;
use crate::native::utils::Normalize;
use rayon::prelude::*;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::thread::available_parallelism;
use std::{cmp, thread};
use std::thread;
use crate::native::logger::enable_logger;
use crate::native::project_graph::utils::{find_project_for_path, ProjectRootMappings};
@ -16,11 +15,12 @@ use crate::native::types::FileData;
use parking_lot::{Condvar, Mutex};
use tracing::{trace, warn};
use crate::native::walker::nx_walker;
use crate::native::workspace::files_archive::{read_files_archive, write_files_archive};
use crate::native::workspace::files_hashing::{full_files_hash, selective_files_hash};
use crate::native::workspace::types::{
FileMap, NxWorkspaceFilesExternals, ProjectFiles, UpdatedWorkspaceFiles,
};
use crate::native::workspace::{config_files, workspace_files, types::NxWorkspaceFiles};
use crate::native::workspace::{config_files, types::NxWorkspaceFiles, workspace_files};
#[napi]
pub struct WorkspaceContext {
@ -33,7 +33,7 @@ type Files = Vec<(PathBuf, String)>;
struct FilesWorker(Option<Arc<(Mutex<Files>, Condvar)>>);
impl FilesWorker {
fn gather_files(workspace_root: &Path) -> Self {
fn gather_files(workspace_root: &Path, cache_dir: String) -> Self {
if !workspace_root.exists() {
warn!(
"workspace root does not exist: {}",
@ -42,6 +42,8 @@ impl FilesWorker {
return FilesWorker(None);
}
let archived_files = read_files_archive(&cache_dir);
let files_lock = Arc::new((Mutex::new(Vec::new()), Condvar::new()));
let files_lock_clone = Arc::clone(&files_lock);
let workspace_root = workspace_root.to_owned();
@ -50,38 +52,27 @@ impl FilesWorker {
trace!("locking files");
let (lock, cvar) = &*files_lock_clone;
let mut workspace_files = lock.lock();
let files = nx_walker(workspace_root).collect::<Vec<_>>();
let num_parallelism = cmp::max(available_parallelism().map_or(2, |n| n.get()) / 3, 2);
let chunks = files.len() / num_parallelism;
let now = std::time::Instant::now();
let mut files = if chunks < num_parallelism {
files
.iter()
.filter_map(|(full_path, path)| {
hash_file_path(full_path).map(|hash| (path.to_owned(), hash))
})
.collect::<Vec<_>>()
let file_hashes = if let Some(archived_files) = archived_files {
selective_files_hash(&workspace_root, archived_files)
} else {
files
.par_chunks(chunks)
.flat_map_iter(|chunks| {
chunks.iter().filter_map(|(full_path, path)| {
hash_file_path(full_path).map(|hash| (path.to_owned(), hash))
})
})
.collect::<Vec<_>>()
full_files_hash(&workspace_root)
};
let mut files = file_hashes
.iter()
.map(|(path, file_hashed)| (PathBuf::from(path), file_hashed.0.to_owned()))
.collect::<Vec<_>>();
files.par_sort();
trace!("hashed and sorted workspace files in {:?}", now.elapsed());
trace!("hashed and sorted files in {:?}", now.elapsed());
*workspace_files = files;
let files_len = workspace_files.len();
trace!(?files_len, "files retrieved");
cvar.notify_all();
write_files_archive(&cache_dir, file_hashes);
});
FilesWorker(Some(files_lock))
@ -162,7 +153,7 @@ impl FilesWorker {
#[napi]
impl WorkspaceContext {
#[napi(constructor)]
pub fn new(workspace_root: String) -> Self {
pub fn new(workspace_root: String, cache_dir: String) -> Self {
enable_logger();
trace!(?workspace_root);
@ -170,7 +161,7 @@ impl WorkspaceContext {
let workspace_root_path = PathBuf::from(&workspace_root);
WorkspaceContext {
files_worker: FilesWorker::gather_files(&workspace_root_path),
files_worker: FilesWorker::gather_files(&workspace_root_path, cache_dir),
workspace_root,
workspace_root_path,
}
@ -180,8 +171,7 @@ impl WorkspaceContext {
pub fn get_workspace_files(
&self,
project_root_map: HashMap<String, String>,
) -> anyhow::Result<NxWorkspaceFiles>
{
) -> anyhow::Result<NxWorkspaceFiles> {
workspace_files::get_files(project_root_map, self.all_file_data())
.map_err(anyhow::Error::from)
}

View File

@ -0,0 +1,91 @@
use anyhow::anyhow;
use hashbrown::HashMap;
use rkyv::{Archive, Deserialize, Infallible, Serialize};
use std::ops::{Deref, DerefMut};
use std::path::Path;
use tracing::trace;
const NX_FILES_ARCHIVE: &str = "nx_files.nxt";
#[derive(Archive, Serialize, Deserialize, PartialEq, Debug)]
#[archive(check_bytes)]
pub struct NxFileHashed(pub String, pub i64);
#[derive(Archive, Deserialize, Serialize, Debug, PartialEq)]
#[archive(check_bytes)]
pub struct NxFileHashes(HashMap<String, NxFileHashed>);
impl Deref for NxFileHashes {
type Target = HashMap<String, NxFileHashed>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl DerefMut for NxFileHashes {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl FromIterator<(String, NxFileHashed)> for NxFileHashes {
fn from_iter<T: IntoIterator<Item = (String, NxFileHashed)>>(iter: T) -> NxFileHashes {
let mut map = HashMap::with_hasher(Default::default());
map.extend(iter);
NxFileHashes(map)
}
}
pub fn read_files_archive<P: AsRef<Path>>(cache_dir: P) -> Option<NxFileHashes> {
let now = std::time::Instant::now();
let archive_path = cache_dir.as_ref().join(NX_FILES_ARCHIVE);
if !archive_path.exists() {
return None;
}
let bytes = std::fs::read(archive_path)
.map_err(anyhow::Error::from)
.and_then(|bytes| {
// let archived = unsafe { rkyv::archived_root::<NxFilesArchive>(&bytes) };
let archived = rkyv::check_archived_root::<NxFileHashes>(&bytes)
.map_err(|_| anyhow!("invalid archive file"))?;
<ArchivedNxFileHashes as Deserialize<NxFileHashes, Infallible>>::deserialize(
archived,
&mut rkyv::Infallible,
)
.map_err(anyhow::Error::from)
});
match bytes {
Ok(archive) => {
trace!("read archive in {:?}", now.elapsed());
Some(archive)
}
Err(e) => {
trace!("could not read files archive: {:?}", e);
None
}
}
}
pub fn write_files_archive<P: AsRef<Path>>(cache_dir: P, files: NxFileHashes) {
let now = std::time::Instant::now();
let archive_path = cache_dir.as_ref().join(NX_FILES_ARCHIVE);
let result = rkyv::to_bytes::<_, 2048>(&files)
.map_err(anyhow::Error::from)
.and_then(|encoded| {
std::fs::write(archive_path, encoded)?;
Ok(())
});
match result {
Ok(_) => {
trace!("write archive in {:?}", now.elapsed());
}
Err(e) => {
trace!("could not write files archive: {:?}", e);
}
}
}

View File

@ -0,0 +1,162 @@
use std::cmp;
use std::path::Path;
use std::thread::available_parallelism;
use rayon::prelude::*;
use tracing::trace;
use crate::native::hasher::hash_file_path;
use crate::native::walker::{nx_walker, NxFile};
use crate::native::workspace::files_archive::{NxFileHashed, NxFileHashes};
pub fn full_files_hash(workspace_root: &Path) -> NxFileHashes {
let files = nx_walker(workspace_root).collect::<Vec<_>>();
hash_files(files).into_iter().collect()
}
pub fn selective_files_hash(
workspace_root: &Path,
mut archived_files: NxFileHashes,
) -> NxFileHashes {
let files = nx_walker(workspace_root).collect::<Vec<_>>();
let mut archived = vec![];
let mut not_archived = vec![];
let now = std::time::Instant::now();
for file in files {
if let Some(archived_file) = archived_files.remove(&file.normalized_path) {
if archived_file.1 == file.mod_time {
archived.push((file.normalized_path, archived_file));
continue;
}
}
not_archived.push(file);
}
trace!("filtered archive files in {:?}", now.elapsed());
if not_archived.is_empty() {
trace!("no additional files to hash");
return archived.into_iter().collect();
}
archived
.into_iter()
.chain(hash_files(not_archived))
.collect()
}
fn hash_files(files: Vec<NxFile>) -> Vec<(String, NxFileHashed)> {
let num_parallelism = cmp::max(available_parallelism().map_or(2, |n| n.get()) / 3, 2);
let chunks = files.len() / num_parallelism;
let now = std::time::Instant::now();
let files = if chunks < num_parallelism {
files
.into_iter()
.filter_map(|file| {
hash_file_path(&file.full_path)
.map(|hash| (file.normalized_path, NxFileHashed(hash, file.mod_time)))
})
.collect::<Vec<_>>()
} else {
files
.par_chunks(chunks)
.flat_map_iter(|chunks| {
chunks.iter().filter_map(|file| {
hash_file_path(&file.full_path).map(|hash| {
(
file.normalized_path.clone(),
NxFileHashed(hash, file.mod_time),
)
})
})
})
.collect::<Vec<_>>()
};
trace!("hashed workspace files in {:?}", now.elapsed());
files
}
#[cfg(test)]
mod tests {
use assert_fs::prelude::*;
use assert_fs::TempDir;
use crate::native::utils::get_mod_time;
use crate::native::workspace::files_archive::{NxFileHashed, NxFileHashes};
fn setup_fs() -> TempDir {
let temp = TempDir::new().unwrap();
temp.child("test.txt").write_str("content").unwrap();
temp.child("modified.txt").write_str("content").unwrap();
temp.child("foo.txt").write_str("content1").unwrap();
temp.child("bar.txt").write_str("content2").unwrap();
temp.child("baz")
.child("new.txt")
.write_str("content@qux")
.unwrap();
temp
}
#[test]
fn should_selectively_hash_workspace() {
let temp = setup_fs();
let archived_files = vec![
(
String::from("test.txt"),
NxFileHashed(
String::from("hash1"),
get_mod_time(&temp.child("test.txt").metadata().unwrap()),
),
),
(
String::from("foo.txt"),
NxFileHashed(
String::from("hash2"),
get_mod_time(&temp.child("foo.txt").metadata().unwrap()),
),
),
(
String::from("bar.txt"),
NxFileHashed(
String::from("hash3"),
get_mod_time(&temp.child("bar.txt").metadata().unwrap()),
),
),
// this file was modified, so the mod time in the archive should be less than whats on the fs to simulate a write
(
String::from("modified.txt"),
NxFileHashed(
String::from("hash4"),
get_mod_time(&temp.child("modified.txt").metadata().unwrap()) - 10,
),
),
// this file is does not exist on the fs, aka it was deleted
(
String::from("baz/qux.txt"),
NxFileHashed(String::from("hash5"), 0),
),
]
.into_iter()
.collect::<NxFileHashes>();
let hashed_files = super::selective_files_hash(temp.path(), archived_files);
let mut hashed_files = hashed_files
.iter()
.map(|(path, _)| path.as_str())
.collect::<Vec<_>>();
hashed_files.sort();
assert_eq!(
hashed_files,
vec![
"bar.txt",
"baz/new.txt",
"foo.txt",
"modified.txt",
"test.txt"
]
)
}
}

View File

@ -1,5 +1,7 @@
pub mod config_files;
pub mod context;
mod errors;
mod files_archive;
mod files_hashing;
pub mod types;
pub mod workspace_files;

View File

@ -81,13 +81,18 @@ export function readProjectsConfigurationFromProjectGraph(
export async function buildProjectGraphWithoutDaemon() {
const nxJson = readNxJson();
performance.mark('retrieve-project-configurations:start');
const { projects, externalNodes, sourceMaps, projectRootMap } =
await retrieveProjectConfigurations(workspaceRoot, nxJson);
performance.mark('retrieve-project-configurations:end');
performance.mark('retrieve-workspace-files:start');
const { allWorkspaceFiles, fileMap, rustReferences } =
await retrieveWorkspaceFiles(workspaceRoot, projectRootMap);
performance.mark('retrieve-workspace-files:end');
const cacheEnabled = process.env.NX_CACHE_PROJECT_GRAPH !== 'false';
performance.mark('build-project-graph-using-project-file-map:start');
const projectGraph = (
await buildProjectGraphUsingProjectFileMap(
projects,
@ -99,6 +104,7 @@ export async function buildProjectGraphWithoutDaemon() {
cacheEnabled
)
).projectGraph;
performance.mark('build-project-graph-using-project-file-map:end');
writeSourceMaps(sourceMaps);
@ -155,6 +161,21 @@ export async function createProjectGraphAsync(
if (!daemonClient.enabled()) {
try {
const res = await buildProjectGraphWithoutDaemon();
performance.measure(
'create-project-graph-async >> retrieve-project-configurations',
'retrieve-project-configurations:start',
'retrieve-project-configurations:end'
);
performance.measure(
'create-project-graph-async >> retrieve-workspace-files',
'retrieve-workspace-files:start',
'retrieve-workspace-files:end'
);
performance.measure(
'create-project-graph-async >> build-project-graph-using-project-file-map',
'build-project-graph-using-project-file-map:start',
'build-project-graph-using-project-file-map:end'
);
performance.mark('create-project-graph-async:end');
performance.measure(
'create-project-graph-async',

View File

@ -60,6 +60,13 @@ export const cacheDir = cacheDirectory(
readCacheDirectoryProperty(workspaceRoot)
);
export function cacheDirectoryForWorkspace(workspaceRoot: string) {
return cacheDirectory(
workspaceRoot,
readCacheDirectoryProperty(workspaceRoot)
);
}
export const projectGraphCacheDirectory = absolutePath(
workspaceRoot,
process.env.NX_PROJECT_GRAPH_CACHE_DIRECTORY ??

View File

@ -1,6 +1,6 @@
import type { NxWorkspaceFilesExternals, WorkspaceContext } from '../native';
import { performance } from 'perf_hooks';
import { ProjectRootMappings } from '../project-graph/utils/find-project-for-path';
import { cacheDirectoryForWorkspace } from './cache-directory';
let workspaceContext: WorkspaceContext | undefined;
@ -8,7 +8,10 @@ export function setupWorkspaceContext(workspaceRoot: string) {
const { WorkspaceContext } =
require('../native') as typeof import('../native');
performance.mark('workspace-context');
workspaceContext = new WorkspaceContext(workspaceRoot);
workspaceContext = new WorkspaceContext(
workspaceRoot,
cacheDirectoryForWorkspace(workspaceRoot)
);
performance.mark('workspace-context:end');
performance.measure(
'workspace context init',