Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 54500810a0 | |||
| c93bf8f1d2 | |||
| df0b260c7b | |||
| c7fff59f6c | |||
| 0f645d0689 | |||
| 2bbcca5253 | |||
| e5f238a126 | |||
| ac8b4d6f81 | |||
| 561400ae3c | |||
| 9d9beefe7f |
215
Cargo.lock
generated
215
Cargo.lock
generated
@@ -3,15 +3,53 @@
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.4.32"
|
||||
name = "anstream"
|
||||
version = "0.6.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a89bce6054c720275ac2432fbba080a66a2106a44a1b804553930ca6909f4e0"
|
||||
checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
|
||||
dependencies = [
|
||||
"compression-codecs",
|
||||
"compression-core",
|
||||
"futures-core",
|
||||
"pin-project-lite",
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
|
||||
dependencies = [
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -33,21 +71,50 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "compression-codecs"
|
||||
version = "0.4.31"
|
||||
name = "clap"
|
||||
version = "4.5.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef8a506ec4b81c460798f572caead636d57d3d7e940f998160f52bd254bf2d23"
|
||||
checksum = "f4512b90fa68d3a9932cea5184017c5d200f5921df706d45e853537dea51508f"
|
||||
dependencies = [
|
||||
"compression-core",
|
||||
"zstd",
|
||||
"zstd-safe",
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compression-core"
|
||||
version = "0.4.29"
|
||||
name = "clap_builder"
|
||||
version = "4.5.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e47641d3deaf41fb1538ac1f54735925e275eaf3bf4d55c81b137fba797e5cbb"
|
||||
checksum = "0025e98baa12e766c67ba13ff4695a887a1eba19569aad00a472546795bd6730"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.5.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
@@ -55,12 +122,6 @@ version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127"
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.3.4"
|
||||
@@ -73,6 +134,18 @@ dependencies = [
|
||||
"wasip2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.15"
|
||||
@@ -107,6 +180,12 @@ version = "2.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.16"
|
||||
@@ -207,6 +286,12 @@ version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.106"
|
||||
@@ -245,6 +330,12 @@ version = "1.0.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.5.0"
|
||||
@@ -270,7 +361,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -279,6 +370,15 @@ version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
@@ -288,6 +388,71 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.53.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen"
|
||||
version = "0.46.0"
|
||||
@@ -295,10 +460,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
|
||||
|
||||
[[package]]
|
||||
name = "zdiff"
|
||||
name = "zsdiff_all"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"clap",
|
||||
"md5",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
||||
14
Cargo.toml
14
Cargo.toml
@@ -1,13 +1,21 @@
|
||||
[package]
|
||||
name = "zdiff"
|
||||
name = "zsdiff_all"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[[bin]]
|
||||
name = "zsdiff"
|
||||
path = "src/zsdiff.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "zspatch"
|
||||
path = "src/zspatch.rs"
|
||||
|
||||
[dependencies]
|
||||
zstd = { version = "0.13" }
|
||||
async-compression = { version = "0.4", features = ["zstd"] }
|
||||
tokio = { version = "1.48.0", features = ["rt", "rt-multi-thread", "macros"] }
|
||||
tokio = { version = "1.48", features = ["rt", "rt-multi-thread", "macros"] }
|
||||
md5 = "0.8"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
walkdir = "2.5"
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
|
||||
9
Makefile
Normal file
9
Makefile
Normal file
@@ -0,0 +1,9 @@
|
||||
all: build-linux build-win
|
||||
|
||||
build-linux:
|
||||
cargo build --release --target x86_64-unknown-linux-gnu --package zsdiff_all --bin zspatch
|
||||
cargo build --release --target x86_64-unknown-linux-gnu --package zsdiff_all --bin zsdiff
|
||||
|
||||
build-win:
|
||||
cargo build --release --target x86_64-pc-windows-gnu --package zsdiff_all --bin zspatch
|
||||
cargo build --release --target x86_64-pc-windows-gnu --package zsdiff_all --bin zsdiff
|
||||
30
README.md
Normal file
30
README.md
Normal file
@@ -0,0 +1,30 @@
|
||||
# ZsDiff
|
||||
|
||||
A partial update program. Use [Zstandrard](https://github.com/facebook/zstd).
|
||||
|
||||
`zsdiff` products a [filename].zdiff file and [filename].zdiff.md5. It compares ALL file hashes from old with new.
|
||||
|
||||
```
|
||||
Usage: zsdiff [OPTIONS] --filename <FILENAME> --old <OLD> --new <NEW>
|
||||
|
||||
Options:
|
||||
-f, --filename <FILENAME>
|
||||
-c, --compress-level <COMPRESS_LEVEL> [default: 11]
|
||||
-o, --old <OLD>
|
||||
-n, --new <NEW>
|
||||
-h, --help Print help
|
||||
```
|
||||
|
||||
`zspatch` extract files from [filename].zdiff to [dest-dir]. If some file was deleted between old and new, `zspatch`
|
||||
also delete it from [dest-dir].
|
||||
Flag --hash-check compare computed hash of [filename].zdiff and hash from [filename].zdiff.md5
|
||||
|
||||
```
|
||||
Usage: zspatch [OPTIONS] --filename <FILENAME> --dest-dir <DEST_DIR>
|
||||
|
||||
Options:
|
||||
-f, --filename <FILENAME>
|
||||
-d, --dest-dir <DEST_DIR>
|
||||
-h, --hash-check
|
||||
--help Print help
|
||||
```
|
||||
53
src/main.rs
53
src/main.rs
@@ -1,53 +0,0 @@
|
||||
mod utils;
|
||||
mod zdiff;
|
||||
mod zpatch;
|
||||
|
||||
use std::fs;
|
||||
use std::fs::read;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
async fn zdiff(filename: &str, old: &str, new: &str) -> Result<(), io::Error> {
|
||||
let output_filename = &format!("{}.zdiff", filename);
|
||||
let old_hashes = zdiff::walk_dir(old.to_string()).await;
|
||||
let new_hashes = zdiff::walk_dir(new.to_string()).await;
|
||||
let compare_hashes = zdiff::compare_hashes(old_hashes, new_hashes).await;
|
||||
let parts = compare_hashes.to_vec().await;
|
||||
utils::compress_parts(parts, fs::File::create(output_filename)?, 11).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn zpatch(filename: &str, dest_dir: &str) -> Result<(), io::Error> {
|
||||
let filename = &format!("{}.zdiff", filename);
|
||||
let parts = utils::decompress_parts(read(filename)?).await?;
|
||||
let zdiff = zdiff::Zdiff::from_vec(parts).await?;
|
||||
let tmp_dir_name = zpatch::extract_files(&zdiff, filename).await?;
|
||||
for name in zdiff.content.keys().collect::<Vec<&String>>() {
|
||||
let from_path = Path::new(&tmp_dir_name).join(name);
|
||||
let to_path = Path::new(&dest_dir).join(name);
|
||||
// println!("{:?} {:?}", from_path, to_path);
|
||||
fs::create_dir_all(to_path.parent().unwrap())?;
|
||||
fs::copy(from_path, to_path)?;
|
||||
}
|
||||
for file in zdiff.metadata.remove_files {
|
||||
let path = Path::new(&dest_dir).join(file);
|
||||
fs::remove_file(path)?;
|
||||
}
|
||||
|
||||
for (k, hash) in zdiff.metadata.hashes {
|
||||
let path = Path::new(&dest_dir).join(k);
|
||||
println!("path: {:?}", path);
|
||||
let content = read(path)?;
|
||||
let fs_hash = zdiff::get_hash(&content).await;
|
||||
println!("{:?} {:?}", hash, fs_hash);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> io::Result<()> {
|
||||
let filename = "test";
|
||||
zdiff(filename, "test/old", "test/new").await?;
|
||||
zpatch(filename, "old").await?;
|
||||
Ok(())
|
||||
}
|
||||
64
src/utils.rs
64
src/utils.rs
@@ -1,7 +1,69 @@
|
||||
use md5;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::{fs, io};
|
||||
use zstd::{Decoder, Encoder};
|
||||
|
||||
pub async fn compress_parts(input: Vec<Vec<u8>>, output: fs::File, level: i32) {
|
||||
pub struct Zsdiff {
|
||||
pub content: HashMap<String, Vec<u8>>,
|
||||
pub metadata: Metadata,
|
||||
}
|
||||
|
||||
impl Zsdiff {
|
||||
pub async fn from_vec(_data: Vec<Vec<u8>>) -> Result<Self, io::Error> {
|
||||
let mut content = HashMap::new();
|
||||
for part in _data {
|
||||
let filename_size = u32::from_be_bytes(part[0..4].try_into().unwrap()) as usize;
|
||||
let filename = String::from_utf8(part[4..filename_size + 4].to_vec()).unwrap();
|
||||
let cont = part[filename_size + 8..].to_vec();
|
||||
content.insert(filename, cont);
|
||||
}
|
||||
let meta = content.get("metadata.json").unwrap();
|
||||
let metadata: Metadata = serde_json::from_slice(meta.as_slice())?;
|
||||
content.remove("metadata.json");
|
||||
|
||||
Ok(Zsdiff { content, metadata })
|
||||
}
|
||||
|
||||
pub async fn to_vec(&self) -> Vec<Vec<u8>> {
|
||||
let mut parts: Vec<Vec<u8>> = Vec::new();
|
||||
for (filename, content) in &self.content {
|
||||
let filename_size: [u8; 4] = (filename.len() as u32).to_be_bytes();
|
||||
let filename_encoded = vec![filename_size.as_slice(), filename.as_bytes()].concat();
|
||||
|
||||
let content_size: [u8; 4] = (content.len() as u32).to_be_bytes();
|
||||
let content_encoded = vec![content_size.as_slice(), content.as_slice()].concat();
|
||||
parts.push(vec![filename_encoded, content_encoded].concat())
|
||||
}
|
||||
|
||||
let meta = serde_json::to_vec(&self.metadata).unwrap();
|
||||
let meta_filename = "metadata.json";
|
||||
let meta_filename_size = (meta_filename.len() as u32).to_be_bytes();
|
||||
let meta_filename_encoded =
|
||||
vec![meta_filename_size.as_slice(), meta_filename.as_bytes()].concat();
|
||||
|
||||
let meta_size = (meta.len() as u32).to_be_bytes();
|
||||
let meta_encoded = vec![meta_size.as_slice(), meta.as_slice()].concat();
|
||||
parts.push(vec![meta_filename_encoded, meta_encoded].concat());
|
||||
|
||||
parts
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct Metadata {
|
||||
pub(crate) diff_files: Vec<String>,
|
||||
pub hashes: HashMap<String, String>,
|
||||
pub remove_files: Vec<String>,
|
||||
pub remove_folders: Vec<String>,
|
||||
}
|
||||
|
||||
pub async fn get_hash(data: Vec<u8>) -> String {
|
||||
let hash = md5::compute(&data[..]);
|
||||
format!("{:x}", hash)
|
||||
}
|
||||
|
||||
pub async fn compress_parts(input: Vec<Vec<u8>>, output: &fs::File, level: i32) {
|
||||
let mut encoder = Encoder::new(output, level).unwrap();
|
||||
for part in input.iter() {
|
||||
io::copy(&mut &part[..], &mut encoder).unwrap();
|
||||
|
||||
127
src/zdiff.rs
127
src/zdiff.rs
@@ -1,127 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Zdiff {
|
||||
pub content: HashMap<String, Vec<u8>>,
|
||||
pub metadata: Metadata,
|
||||
}
|
||||
|
||||
impl Zdiff {
|
||||
pub async fn from_vec(_data: Vec<Vec<u8>>) -> Result<Self, std::io::Error> {
|
||||
let mut content = HashMap::new();
|
||||
for part in _data {
|
||||
let filename_size = u32::from_be_bytes(part[0..4].try_into().unwrap()) as usize;
|
||||
let filename = String::from_utf8(part[4..filename_size + 4].to_vec()).unwrap();
|
||||
let cont = part[filename_size + 8..].to_vec();
|
||||
content.insert(filename, cont);
|
||||
}
|
||||
let meta = content.get("metadata.json").unwrap();
|
||||
let metadata: Metadata = serde_json::from_slice(meta.as_slice())?;
|
||||
content.remove("metadata.json");
|
||||
|
||||
Ok(Zdiff { content, metadata })
|
||||
}
|
||||
|
||||
pub async fn to_vec(&self) -> Vec<Vec<u8>> {
|
||||
let mut parts: Vec<Vec<u8>> = Vec::new();
|
||||
for (filename, content) in &self.content {
|
||||
let filename_size: [u8; 4] = (filename.len() as u32).to_be_bytes();
|
||||
let filename_encoded = vec![filename_size.as_slice(), filename.as_bytes()].concat();
|
||||
|
||||
let content_size: [u8; 4] = (content.len() as u32).to_be_bytes();
|
||||
let content_encoded = vec![content_size.as_slice(), content.as_slice()].concat();
|
||||
parts.push(vec![filename_encoded, content_encoded].concat())
|
||||
}
|
||||
|
||||
let meta = serde_json::to_vec(&self.metadata).unwrap();
|
||||
let meta_filename = "metadata.json";
|
||||
let meta_filename_size = (meta_filename.len() as u32).to_be_bytes();
|
||||
let meta_filename_encoded =
|
||||
vec![meta_filename_size.as_slice(), meta_filename.as_bytes()].concat();
|
||||
|
||||
let meta_size = (meta.len() as u32).to_be_bytes();
|
||||
let meta_encoded = vec![meta_size.as_slice(), meta.as_slice()].concat();
|
||||
parts.push(vec![meta_filename_encoded, meta_encoded].concat());
|
||||
|
||||
parts
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct Metadata {
|
||||
diff_files: Vec<String>,
|
||||
pub hashes: HashMap<String, String>,
|
||||
pub remove_files: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct FileInfo {
|
||||
path: String,
|
||||
relative_path: String, // Without dir prefix
|
||||
hash: String,
|
||||
}
|
||||
|
||||
pub async fn get_hash(data: &Vec<u8>) -> String {
|
||||
let hash = md5::compute(&data[..]);
|
||||
format!("{:x}", hash)
|
||||
}
|
||||
|
||||
pub async fn walk_dir(dir: String) -> HashMap<String, FileInfo> {
|
||||
let mut hash_list: HashMap<String, FileInfo> = HashMap::new();
|
||||
for e in WalkDir::new(&dir) {
|
||||
let e = e.unwrap();
|
||||
let path = e.path();
|
||||
if path.is_dir() {
|
||||
continue;
|
||||
}
|
||||
let content = fs::read(path).unwrap();
|
||||
let hash = get_hash(&content).await;
|
||||
// let filename = path.file_name().unwrap().to_str().unwrap().to_string();
|
||||
let path_str = path.display().to_string();
|
||||
let file_info = FileInfo {
|
||||
relative_path: path_str[dir.len() + 1..].to_string(),
|
||||
path: path_str,
|
||||
hash: hash.clone(),
|
||||
};
|
||||
hash_list.insert(hash, file_info);
|
||||
}
|
||||
hash_list
|
||||
}
|
||||
|
||||
pub async fn compare_hashes(
|
||||
old: HashMap<String, FileInfo>,
|
||||
new: HashMap<String, FileInfo>,
|
||||
) -> Zdiff {
|
||||
let mut diff_files: HashMap<String, Vec<u8>> = HashMap::new();
|
||||
let mut remove_files: Vec<String> = vec![];
|
||||
let mut hashes: HashMap<String, String> = HashMap::new();
|
||||
for (_, info) in &old {
|
||||
remove_files.push(info.relative_path.clone());
|
||||
}
|
||||
|
||||
for (new_hash, new_fileinfo) in &new {
|
||||
let old_fileinfo = old.get(new_hash);
|
||||
remove_files.retain(|filename| !filename.eq(&new_fileinfo.relative_path));
|
||||
|
||||
if old_fileinfo.is_none() {
|
||||
let path = new_fileinfo.relative_path.clone();
|
||||
diff_files.insert(path.clone(), fs::read(new_fileinfo.path.clone()).unwrap());
|
||||
hashes.insert(
|
||||
new_fileinfo.relative_path.clone(),
|
||||
new_fileinfo.hash.clone(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Zdiff {
|
||||
content: diff_files.clone(),
|
||||
metadata: Metadata {
|
||||
diff_files: diff_files.keys().cloned().collect(),
|
||||
hashes,
|
||||
remove_files,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
use crate::zdiff::Zdiff;
|
||||
use crate::zpatch;
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
|
||||
pub async fn create_tmp_dir(dir_name: String) -> Result<String, std::io::Error> {
|
||||
let name = format!("{}.tmp", dir_name);
|
||||
fs::remove_dir_all(name.clone()).map_err(|_| std::io::ErrorKind::NotFound)?;
|
||||
fs::DirBuilder::new().create(name.clone())?;
|
||||
Ok(name)
|
||||
}
|
||||
|
||||
pub async fn extract_files(zdiff: &Zdiff, filename: &String) -> Result<String, std::io::Error> {
|
||||
let tmp_dir_name = create_tmp_dir(filename.to_string()).await?;
|
||||
let path = Path::new(&tmp_dir_name);
|
||||
fs::remove_dir_all(path)?;
|
||||
for (f, c) in zdiff.content.iter() {
|
||||
let filepath = path.join(f);
|
||||
fs::create_dir_all(filepath.parent().unwrap())?;
|
||||
fs::File::create(&filepath)?.write_all(c)?;
|
||||
}
|
||||
Ok(tmp_dir_name)
|
||||
}
|
||||
142
src/zsdiff.rs
Normal file
142
src/zsdiff.rs
Normal file
@@ -0,0 +1,142 @@
|
||||
mod utils;
|
||||
|
||||
use clap::Parser;
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::format;
|
||||
use std::io::Write;
|
||||
use std::{fs, io, time};
|
||||
use utils::{Metadata, Zsdiff, get_hash};
|
||||
use walkdir::WalkDir;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FileInfo {
|
||||
path: String,
|
||||
relative_path: String, // Without dir prefix
|
||||
hash: String,
|
||||
is_dir: bool,
|
||||
}
|
||||
|
||||
async fn walk_dir(dir: String) -> HashMap<String, FileInfo> {
|
||||
let mut hash_list: HashMap<String, FileInfo> = HashMap::new();
|
||||
for e in WalkDir::new(&dir) {
|
||||
let e = e.unwrap();
|
||||
let path = e.path();
|
||||
if path.display().to_string().eq(&dir) {
|
||||
continue;
|
||||
}
|
||||
let content: Vec<u8>;
|
||||
if path.is_dir() {
|
||||
let path_str = path.display().to_string();
|
||||
content = path_str.into_bytes();
|
||||
} else {
|
||||
content = fs::read(path).unwrap();
|
||||
}
|
||||
let hash = get_hash(content).await;
|
||||
let path_str = path.display().to_string();
|
||||
let file_info = FileInfo {
|
||||
relative_path: path_str[dir.len() + 1..].to_string(),
|
||||
path: path_str,
|
||||
hash: hash.clone(),
|
||||
is_dir: path.is_dir(),
|
||||
};
|
||||
hash_list.entry(hash).or_insert(file_info);
|
||||
}
|
||||
hash_list
|
||||
}
|
||||
|
||||
async fn compare_hashes(old: HashMap<String, FileInfo>, new: HashMap<String, FileInfo>) -> Zsdiff {
|
||||
let mut diff_files: HashMap<String, Vec<u8>> = HashMap::new();
|
||||
let mut remove_files: Vec<String> = vec![];
|
||||
let mut remove_folders: Vec<String> = vec![];
|
||||
let mut hashes: HashMap<String, String> = HashMap::new();
|
||||
for (_, info) in &old {
|
||||
if info.is_dir {
|
||||
remove_folders.push(info.relative_path.clone());
|
||||
} else {
|
||||
remove_files.push(info.relative_path.clone());
|
||||
}
|
||||
}
|
||||
|
||||
for (new_hash, new_fileinfo) in &new {
|
||||
let old_fileinfo = old.get(new_hash);
|
||||
if new_fileinfo.is_dir {
|
||||
remove_folders.retain(|filename| !filename.eq(&new_fileinfo.relative_path));
|
||||
} else {
|
||||
remove_files.retain(|filename| !filename.eq(&new_fileinfo.relative_path));
|
||||
}
|
||||
|
||||
if new_fileinfo.is_dir {
|
||||
continue;
|
||||
}
|
||||
|
||||
if old_fileinfo.is_none() {
|
||||
let path = new_fileinfo.relative_path.clone();
|
||||
diff_files.insert(path.clone(), fs::read(new_fileinfo.path.clone()).unwrap());
|
||||
hashes.insert(
|
||||
new_fileinfo.relative_path.clone(),
|
||||
new_fileinfo.hash.clone(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Zsdiff {
|
||||
content: diff_files.clone(),
|
||||
metadata: Metadata {
|
||||
diff_files: diff_files.keys().cloned().collect(),
|
||||
hashes,
|
||||
remove_files,
|
||||
remove_folders,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn zsdiff(
|
||||
filename: String,
|
||||
old: String,
|
||||
new: String,
|
||||
level: i32,
|
||||
) -> Result<(), io::Error> {
|
||||
let output_filename = &format!("{}.zdiff", filename);
|
||||
let old_hashes = walk_dir(old).await;
|
||||
let new_hashes = walk_dir(new).await;
|
||||
let compare_hashes = compare_hashes(old_hashes, new_hashes).await;
|
||||
let parts = compare_hashes.to_vec().await;
|
||||
let mut size_before = 0;
|
||||
for p in &parts {
|
||||
size_before += p.len();
|
||||
}
|
||||
let now = time::Instant::now();
|
||||
utils::compress_parts(parts, &fs::File::create(output_filename)?, level).await;
|
||||
let output_data = fs::read(output_filename)?;
|
||||
let size_after = output_data.len();
|
||||
let output_hash = format!("{} {}", get_hash(output_data).await, output_filename);
|
||||
fs::File::create(format!("{}.md5", output_filename))?.write_all(output_hash.as_bytes())?;
|
||||
let elapsed = now.elapsed();
|
||||
println!("Zsdiff hash: {}", output_hash);
|
||||
println!("Size before: {:.1?}KB", size_before / 1024);
|
||||
println!("Size after: {:.1?}KB", size_after / 1024);
|
||||
println!(
|
||||
"Compress ratio: {:.2?}%",
|
||||
size_after as f64 / size_before as f64 * 100.0
|
||||
);
|
||||
print!("Time elapsed: {:.2?}", elapsed);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Parser)]
|
||||
struct Args {
|
||||
#[arg(short, long)]
|
||||
filename: String,
|
||||
#[arg(short, long, default_value_t = 11)]
|
||||
compress_level: i32,
|
||||
#[arg(short, long)]
|
||||
old: String,
|
||||
#[arg(short, long)]
|
||||
new: String,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> io::Result<()> {
|
||||
let args = Args::parse();
|
||||
zsdiff(args.filename, args.old, args.new, args.compress_level).await
|
||||
}
|
||||
105
src/zspatch.rs
Normal file
105
src/zspatch.rs
Normal file
@@ -0,0 +1,105 @@
|
||||
mod utils;
|
||||
|
||||
use clap::Parser;
|
||||
use std::fs::read;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::{fs, io, time};
|
||||
use utils::Zsdiff;
|
||||
|
||||
async fn create_tmp_dir(dir_name: String) -> Result<String, io::Error> {
|
||||
let name = format!("{}.tmp", dir_name);
|
||||
fs::remove_dir_all(name.clone()).ok();
|
||||
fs::DirBuilder::new().create(name.clone())?;
|
||||
Ok(name)
|
||||
}
|
||||
|
||||
async fn load_file(filename: String) -> Result<Zsdiff, io::Error> {
|
||||
let filename = &format!("{}.zdiff", filename);
|
||||
let parts = utils::decompress_parts(read(filename)?).await?;
|
||||
Ok(Zsdiff::from_vec(parts).await?)
|
||||
}
|
||||
|
||||
async fn extract_files(zsdiff: &Zsdiff, filename: String) -> Result<String, io::Error> {
|
||||
let tmp_dir_name = create_tmp_dir(filename.to_string()).await?;
|
||||
let path = Path::new(&tmp_dir_name);
|
||||
fs::remove_dir_all(path).ok();
|
||||
for (f, c) in zsdiff.content.iter() {
|
||||
let filepath = path.join(f);
|
||||
fs::create_dir_all(filepath.parent().unwrap())?;
|
||||
fs::File::create(&filepath)?.write_all(c)?;
|
||||
}
|
||||
Ok(tmp_dir_name)
|
||||
}
|
||||
|
||||
async fn check_hash(filename: String) -> Result<(), io::Error> {
|
||||
let file_data = read(format!("{}.zdiff", filename))?;
|
||||
let hash_file = String::from_utf8(read(format!("{}.zdiff.md5", filename))?).unwrap();
|
||||
let hash = utils::get_hash(file_data).await;
|
||||
if !hash_file.split(" ").next().unwrap().eq(&hash) {
|
||||
return Err(io::Error::new(io::ErrorKind::Other, "Hash mismatch"));
|
||||
}
|
||||
println!("Zsdiff hash: {}", hash);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn zspatch(filename: String, dest_dir: String) -> Result<(), io::Error> {
|
||||
let diff = load_file(filename.clone()).await?;
|
||||
let tmp_dir_name = extract_files(&diff, filename).await?;
|
||||
let now = time::Instant::now();
|
||||
for name in diff.content.keys().collect::<Vec<&String>>() {
|
||||
let from_path = Path::new(&tmp_dir_name).join(name);
|
||||
let to_path = Path::new(&dest_dir).join(name);
|
||||
fs::create_dir_all(to_path.parent().unwrap())?;
|
||||
fs::copy(from_path, to_path)?;
|
||||
}
|
||||
|
||||
for file in diff.metadata.remove_files {
|
||||
let path = Path::new(&dest_dir).join(file);
|
||||
fs::remove_file(path).ok();
|
||||
}
|
||||
|
||||
for folder in diff.metadata.remove_folders {
|
||||
let path = Path::new(&dest_dir).join(folder);
|
||||
fs::remove_dir_all(path).ok();
|
||||
}
|
||||
|
||||
for (k, hash) in diff.metadata.hashes {
|
||||
let path = Path::new(&dest_dir).join(k);
|
||||
let content = read(path)?;
|
||||
let fs_hash = utils::get_hash(content).await;
|
||||
if !fs_hash.eq(&hash) {
|
||||
Err(io::Error::new(io::ErrorKind::Other, "Hash mismatch"))?
|
||||
}
|
||||
}
|
||||
fs::remove_dir_all(tmp_dir_name).ok();
|
||||
println!("Patching done!");
|
||||
println!("Elapsed time: {:.2?}", now.elapsed());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Parser)]
|
||||
struct Args {
|
||||
#[arg(short, long)]
|
||||
filename: String,
|
||||
#[arg(short, long)]
|
||||
dest_dir: String,
|
||||
#[arg(short, long)]
|
||||
metadata: bool,
|
||||
#[arg(short, long)]
|
||||
hash_check: bool,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> io::Result<()> {
|
||||
let args = Args::parse();
|
||||
if args.hash_check {
|
||||
check_hash(args.filename.clone()).await?;
|
||||
}
|
||||
if args.metadata {
|
||||
let diff = load_file(args.filename).await?;
|
||||
println!("{}", serde_json::to_string(&diff.metadata)?);
|
||||
return Ok(());
|
||||
}
|
||||
zspatch(args.filename, args.dest_dir).await
|
||||
}
|
||||
Reference in New Issue
Block a user