Skip to content

Commit 29e0a7f

Browse files
jerrykingxyzjerrykingxyz
authored
refactor: differentiate snapshot strategies by dependency type (#12805)
* refactor: differentiate snapshot strategies by dependency type * fix: ci * feat: independent snapshot scope for different dependency types
1 parent 6a174db commit 29e0a7f

File tree

12 files changed

+319
-251
lines changed

12 files changed

+319
-251
lines changed

‎crates/rspack_binding_api/src/compilation/dependencies.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ impl JsDependencies {
3737
self
3838
.compilation
3939
.file_dependencies()
40-
.2
40+
.3
4141
.map(|i| i.to_string_lossy().to_string())
4242
.collect()
4343
}
@@ -65,7 +65,7 @@ impl JsDependencies {
6565
self
6666
.compilation
6767
.context_dependencies()
68-
.2
68+
.3
6969
.map(|i| i.to_string_lossy().to_string())
7070
.collect()
7171
}
@@ -93,7 +93,7 @@ impl JsDependencies {
9393
self
9494
.compilation
9595
.missing_dependencies()
96-
.2
96+
.3
9797
.map(|i| i.to_string_lossy().to_string())
9898
.collect()
9999
}
@@ -121,7 +121,7 @@ impl JsDependencies {
121121
self
122122
.compilation
123123
.build_dependencies()
124-
.2
124+
.3
125125
.map(|i| i.to_string_lossy().to_string())
126126
.collect()
127127
}

‎crates/rspack_core/src/cache/persistent/build_dependencies/mod.rs‎

Lines changed: 28 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,7 @@ use rustc_hash::FxHashSet as HashSet;
99

1010
use self::helper::{Helper, is_node_package_path};
1111
use super::{
12-
codec::CacheCodec,
13-
snapshot::{Snapshot, SnapshotOptions},
12+
snapshot::{Snapshot, SnapshotScope},
1413
storage::Storage,
1514
};
1615

@@ -30,29 +29,22 @@ pub struct BuildDeps {
3029
/// The next time the add method is called, this path will be additionally added.
3130
pending: ArcPathSet,
3231
/// The snapshot which is used to save build dependencies.
33-
snapshot: Snapshot,
32+
snapshot: Arc<Snapshot>,
3433
storage: Arc<dyn Storage>,
3534
fs: Arc<dyn ReadableFileSystem>,
3635
}
3736

3837
impl BuildDeps {
3938
pub fn new(
4039
options: &BuildDepsOptions,
41-
snapshot_options: &SnapshotOptions,
4240
fs: Arc<dyn ReadableFileSystem>,
41+
snapshot: Arc<Snapshot>,
4342
storage: Arc<dyn Storage>,
44-
codec: Arc<CacheCodec>,
4543
) -> Self {
4644
Self {
4745
added: Default::default(),
4846
pending: options.iter().map(|v| ArcPath::from(v.as_path())).collect(),
49-
snapshot: Snapshot::new_with_scope(
50-
SCOPE,
51-
snapshot_options.clone(),
52-
fs.clone(),
53-
storage.clone(),
54-
codec,
55-
),
47+
snapshot,
5648
storage,
5749
fs,
5850
}
@@ -84,16 +76,21 @@ impl BuildDeps {
8476
}
8577
}
8678

87-
self.snapshot.add(new_deps.into_iter()).await;
79+
self
80+
.snapshot
81+
.add(SnapshotScope::BUILD, new_deps.into_iter())
82+
.await;
8883
helper.into_warnings()
8984
}
9085

9186
/// Validate build dependencies
9287
///
9388
/// If any build dependencies have changed, this method will reset storage.
9489
pub async fn validate(&mut self) -> Result<()> {
95-
let (_, modified_files, removed_files, no_changed_files) =
96-
self.snapshot.calc_modified_paths().await?;
90+
let (_, modified_files, removed_files, no_changed_files) = self
91+
.snapshot
92+
.calc_modified_paths(SnapshotScope::BUILD)
93+
.await?;
9794

9895
if !modified_files.is_empty() || !removed_files.is_empty() {
9996
self.storage.reset().await;
@@ -116,11 +113,16 @@ mod test {
116113
use rspack_storage::Storage;
117114

118115
use super::{
119-
super::{codec::CacheCodec, storage::MemoryStorage},
120-
BuildDeps, SCOPE, SnapshotOptions,
116+
super::{
117+
codec::CacheCodec,
118+
snapshot::{Snapshot, SnapshotOptions, SnapshotScope},
119+
storage::MemoryStorage,
120+
},
121+
BuildDeps,
121122
};
122123
#[tokio::test]
123124
async fn build_dependencies_test() {
125+
let scope = SnapshotScope::BUILD.name();
124126
let fs = Arc::new(MemoryFileSystem::default());
125127
fs.create_dir_all("/configs/test".into()).await.unwrap();
126128
fs.write("/configs/a.js".into(), r#"console.log('a')"#.as_bytes())
@@ -155,28 +157,21 @@ mod test {
155157
.unwrap();
156158

157159
let options = vec![PathBuf::from("/index.js"), PathBuf::from("/configs")];
158-
let snapshot_options = SnapshotOptions::default();
159160
let storage = Arc::new(MemoryStorage::default());
160161
let codec = Arc::new(CacheCodec::new(None));
161-
let mut build_deps = BuildDeps::new(
162-
&options,
163-
&snapshot_options,
162+
let snapshot = Arc::new(Snapshot::new(
163+
SnapshotOptions::default(),
164164
fs.clone(),
165165
storage.clone(),
166-
codec.clone(),
167-
);
166+
codec,
167+
));
168+
let mut build_deps = BuildDeps::new(&options, fs.clone(), snapshot.clone(), storage.clone());
168169
let warnings = build_deps.add(vec![].into_iter()).await;
169170
assert_eq!(warnings.len(), 1);
170-
let data = storage.load(SCOPE).await.expect("should load success");
171+
let data = storage.load(scope).await.expect("should load success");
171172
assert_eq!(data.len(), 9);
172173

173-
let mut build_deps = BuildDeps::new(
174-
&options,
175-
&snapshot_options,
176-
fs.clone(),
177-
storage.clone(),
178-
codec,
179-
);
174+
let mut build_deps = BuildDeps::new(&options, fs.clone(), snapshot.clone(), storage.clone());
180175
fs.write("/b.js".into(), r#"require("./c")"#.as_bytes())
181176
.await
182177
.unwrap();
@@ -185,11 +180,11 @@ mod test {
185180
.await
186181
.expect("should validate success");
187182

188-
let data = storage.load(SCOPE).await.expect("should load success");
183+
let data = storage.load(scope).await.expect("should load success");
189184
assert_eq!(data.len(), 0);
190185
let warnings = build_deps.add(vec![].into_iter()).await;
191186
assert_eq!(warnings.len(), 0);
192-
let data = storage.load(SCOPE).await.expect("should load success");
187+
let data = storage.load(scope).await.expect("should load success");
193188
assert_eq!(data.len(), 10);
194189
}
195190
}

‎crates/rspack_core/src/cache/persistent/mod.rs‎

Lines changed: 69 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use self::{
2222
build_dependencies::{BuildDeps, BuildDepsOptions},
2323
codec::CacheCodec,
2424
occasion::{MakeOccasion, MetaOccasion},
25-
snapshot::{Snapshot, SnapshotOptions},
25+
snapshot::{Snapshot, SnapshotOptions, SnapshotScope},
2626
storage::{Storage, StorageOptions, create_storage},
2727
};
2828
use super::Cache;
@@ -46,7 +46,7 @@ pub struct PersistentCacheOptions {
4646
pub struct PersistentCache {
4747
initialized: bool,
4848
build_deps: BuildDeps,
49-
snapshot: Snapshot,
49+
snapshot: Arc<Snapshot>,
5050
make_occasion: MakeOccasion,
5151
meta_occasion: MetaOccasion,
5252
async_mode: bool,
@@ -80,22 +80,22 @@ impl PersistentCache {
8080
hex::encode(hasher.finish().to_ne_bytes())
8181
};
8282
let storage = create_storage(option.storage.clone(), version, intermediate_filesystem);
83+
let snapshot = Arc::new(Snapshot::new(
84+
option.snapshot.clone(),
85+
input_filesystem.clone(),
86+
storage.clone(),
87+
codec.clone(),
88+
));
8389

8490
Self {
8591
initialized: false,
8692
build_deps: BuildDeps::new(
8793
&option.build_dependencies,
88-
&option.snapshot,
89-
input_filesystem.clone(),
90-
storage.clone(),
91-
codec.clone(),
92-
),
93-
snapshot: Snapshot::new(
94-
option.snapshot.clone(),
9594
input_filesystem,
95+
snapshot.clone(),
9696
storage.clone(),
97-
codec.clone(),
9897
),
98+
snapshot,
9999
make_occasion: MakeOccasion::new(storage.clone(), codec.clone()),
100100
meta_occasion: MetaOccasion::new(storage.clone(), codec),
101101
warnings: Default::default(),
@@ -147,14 +147,34 @@ impl Cache for PersistentCache {
147147
// rebuild will pass modified_files and removed_files from js side,
148148
// so only calculate them when build.
149149
if !compilation.is_rebuild {
150-
let (is_hot_start, modified_paths, removed_paths, _) =
151-
match self.snapshot.calc_modified_paths().await {
152-
Ok(res) => res,
150+
let mut is_hot_start = false;
151+
let mut modified_paths = ArcPathSet::default();
152+
let mut removed_paths = ArcPathSet::default();
153+
let data = vec![
154+
self.snapshot.calc_modified_paths(SnapshotScope::FILE).await,
155+
self
156+
.snapshot
157+
.calc_modified_paths(SnapshotScope::CONTEXT)
158+
.await,
159+
self
160+
.snapshot
161+
.calc_modified_paths(SnapshotScope::MISSING)
162+
.await,
163+
];
164+
for item in data {
165+
match item {
166+
Ok((a, b, c, _)) => {
167+
is_hot_start = is_hot_start || a;
168+
modified_paths.extend(b);
169+
removed_paths.extend(c);
170+
}
153171
Err(err) => {
154172
self.warnings.push(err.to_string());
155173
return false;
156174
}
157175
};
176+
}
177+
158178
tracing::debug!("cache::snapshot recovery {modified_paths:?} {removed_paths:?}",);
159179
compilation.modified_files.extend(modified_paths);
160180
compilation.removed_files.extend(removed_paths);
@@ -169,31 +189,43 @@ impl Cache for PersistentCache {
169189

170190
// save snapshot
171191
// TODO add a all_dependencies to collect dependencies
172-
let (_, file_added, file_removed) = compilation.file_dependencies();
173-
let (_, context_added, context_removed) = compilation.context_dependencies();
174-
let (_, missing_added, missing_removed) = compilation.missing_dependencies();
175-
let (_, build_added, _) = compilation.build_dependencies();
176-
let modified_paths: ArcPathSet = compilation
177-
.modified_files
178-
.iter()
179-
.chain(file_added)
180-
.chain(missing_added)
181-
.chain(context_added)
182-
.cloned()
183-
.collect();
184-
let removed_paths: ArcPathSet = compilation
185-
.removed_files
186-
.iter()
187-
.chain(file_removed)
188-
.chain(context_removed)
189-
.chain(missing_removed)
190-
.cloned()
191-
.collect();
192-
self.snapshot.remove(removed_paths.into_iter());
193-
self.snapshot.add(modified_paths.into_iter()).await;
192+
let (_, file_added, file_updated, file_removed) = compilation.file_dependencies();
193+
let (_, context_added, context_updated, context_removed) = compilation.context_dependencies();
194+
let (_, missing_added, missing_updated, missing_removed) = compilation.missing_dependencies();
195+
let (_, build_added, build_updated, _) = compilation.build_dependencies();
196+
self
197+
.snapshot
198+
.remove(SnapshotScope::FILE, file_removed.cloned());
199+
self
200+
.snapshot
201+
.remove(SnapshotScope::CONTEXT, context_removed.cloned());
202+
self
203+
.snapshot
204+
.remove(SnapshotScope::MISSING, missing_removed.cloned());
205+
self
206+
.snapshot
207+
.add(SnapshotScope::FILE, file_added.chain(file_updated).cloned())
208+
.await;
209+
self
210+
.snapshot
211+
.add(
212+
SnapshotScope::CONTEXT,
213+
context_added.chain(context_updated).cloned(),
214+
)
215+
.await;
194216
self
195-
.warnings
196-
.extend(self.build_deps.add(build_added.cloned()).await);
217+
.snapshot
218+
.add(
219+
SnapshotScope::MISSING,
220+
missing_added.chain(missing_updated).cloned(),
221+
)
222+
.await;
223+
self.warnings.extend(
224+
self
225+
.build_deps
226+
.add(build_added.chain(build_updated).cloned())
227+
.await,
228+
);
197229

198230
self.save().await;
199231

0 commit comments

Comments
 (0)