From 050a844bc2dede4c9d818ce7a587b5b04e390d61 Mon Sep 17 00:00:00 2001 From: innocentzero Date: Tue, 23 Jan 2024 20:42:54 +0530 Subject: [PATCH 1/7] feat(pipx support): Add support for pipx This is a series of commits intended to add support for pipx in pacdef. WIP. Signed-off-by: innocentzero --- .../pacdef_core/src/backend/actual/python.rs | 27 ++++++++++++++++--- crates/pacdef_core/src/config.rs | 4 +++ crates/pacdef_core/src/core.rs | 7 +++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/crates/pacdef_core/src/backend/actual/python.rs b/crates/pacdef_core/src/backend/actual/python.rs index 8ba3438..f309600 100644 --- a/crates/pacdef_core/src/backend/actual/python.rs +++ b/crates/pacdef_core/src/backend/actual/python.rs @@ -12,6 +12,7 @@ use crate::{Group, Package}; #[derive(Debug, Clone)] pub struct Python { + pub(crate) binary: String, pub(crate) packages: HashSet, } @@ -29,14 +30,24 @@ const SUPPORTS_AS_DEPENDENCY: bool = false; impl Backend for Python { impl_backend_constants!(); + fn get_binary(&self) -> Text { + let r#box = self.binary.clone().into_boxed_str(); + Box::leak(r#box) + } + fn get_all_installed_packages(&self) -> Result> { - let output = run_pip_command(&["list", "--format", "json", "--user"])?; + let mut cmd = Command::new(self.get_binary()); + let output = run_pip_command(&mut cmd, self.get_switches_runtime())?; extract_pacdef_packages(output) } fn get_explicitly_installed_packages(&self) -> Result> { - let output = run_pip_command(&["list", "--format", "json", "--not-required", "--user"])?; + let mut cmd = Command::new(self.get_binary()); + let output = run_pip_command( + &mut cmd, + &["list", "--format", "json", "--not-required", "--user"], + )?; extract_pacdef_packages(output) } @@ -46,8 +57,7 @@ impl Backend for Python { } } -fn run_pip_command(args: &[&str]) -> Result { - let mut cmd = Command::new(BINARY); +fn run_pip_command(cmd: &mut Command, args: &[&str]) -> Result { cmd.args(args); let output = String::from_utf8(cmd.output()?.stdout)?; let val: Value = serde_json::from_str(&output)?; @@ -57,9 +67,18 @@ fn run_pip_command(args: &[&str]) -> Result { impl Python { pub(crate) fn new() -> Self { Self { + binary: BINARY.to_string(), packages: HashSet::new(), } } + + fn get_switches_runtime(&self) -> Switches { + if self.get_binary().eq("pip") { + &["list", "--format", "json", "--user"] + } else { + &["list", "--json"] + } + } } fn extract_pacdef_packages(value: Value) -> Result> { diff --git a/crates/pacdef_core/src/config.rs b/crates/pacdef_core/src/config.rs index dbbec27..dbcafc1 100644 --- a/crates/pacdef_core/src/config.rs +++ b/crates/pacdef_core/src/config.rs @@ -24,6 +24,9 @@ pub struct Config { /// Backends the user does not want to use even though the binary exists. #[serde(default)] pub disabled_backends: Vec, + /// Choose whether to use pipx instead of pip for python package management + #[serde(default)] + pub pip_binary: String, } fn yes() -> bool { @@ -86,6 +89,7 @@ impl Default for Config { flatpak_systemwide: true, warn_not_symlinks: true, disabled_backends: vec![], + pip_binary: "pip".into(), } } } diff --git a/crates/pacdef_core/src/core.rs b/crates/pacdef_core/src/core.rs index ccf2839..b79b6e8 100644 --- a/crates/pacdef_core/src/core.rs +++ b/crates/pacdef_core/src/core.rs @@ -140,6 +140,13 @@ impl Pacdef { { flatpak.systemwide = self.config.flatpak_systemwide; } + + if let Some(python) = backend + .as_any_mut() + .downcast_mut::() + { + python.binary = self.config.pip_binary.clone(); + } } fn install_packages(&mut self, noconfirm: bool) -> Result<()> { From 242af523b4c513a93a1bf7ed1b88deee5bd6f6b5 Mon Sep 17 00:00:00 2001 From: innocentzero Date: Wed, 24 Jan 2024 20:21:05 +0530 Subject: [PATCH 2/7] FIX iterate through json generated by pipx While it is now able to iterate through JSON created by pipx, it is unable to read the JSON with the error `EOF while parsing a value at line 1 column 0` Signed-off-by: innocentzero --- .../pacdef_core/src/backend/actual/python.rs | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/pacdef_core/src/backend/actual/python.rs b/crates/pacdef_core/src/backend/actual/python.rs index f309600..68f0636 100644 --- a/crates/pacdef_core/src/backend/actual/python.rs +++ b/crates/pacdef_core/src/backend/actual/python.rs @@ -39,7 +39,7 @@ impl Backend for Python { let mut cmd = Command::new(self.get_binary()); let output = run_pip_command(&mut cmd, self.get_switches_runtime())?; - extract_pacdef_packages(output) + extract_pacdef_packages_pipx(output) } fn get_explicitly_installed_packages(&self) -> Result> { @@ -49,7 +49,7 @@ impl Backend for Python { &["list", "--format", "json", "--not-required", "--user"], )?; - extract_pacdef_packages(output) + extract_pacdef_packages_pipx(output) } fn make_dependency(&self, _packages: &[Package]) -> Result { @@ -73,10 +73,10 @@ impl Python { } fn get_switches_runtime(&self) -> Switches { - if self.get_binary().eq("pip") { - &["list", "--format", "json", "--user"] - } else { - &["list", "--json"] + match self.get_binary() { + "pip" => &["list", "--format", "json", "--user"], + "pipx" => &["list", "--json"], + _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), } } } @@ -91,3 +91,17 @@ fn extract_pacdef_packages(value: Value) -> Result> { .collect(); Ok(result) } + +fn extract_pacdef_packages_pipx(value: Value) -> Result> { + let result = value["venvs"] + .as_object() + .context("getting inner json object")? + .iter() + .map(|(name, _)| { + println!("{name}"); + Package::from(name.as_str()) + }) + // .map(|(name, _)| Package::from(name.as_str())) + .collect(); + Ok(result) +} From 2a7c5d528802b1f2f9927e20c63551cbc9f6ae4f Mon Sep 17 00:00:00 2001 From: innocentzero Date: Thu, 25 Jan 2024 11:52:16 +0530 Subject: [PATCH 3/7] FIX refactored switches and output parsing Removed inlined switches to a separate function for both explicit installs and all installs and added a match statement to parse the outputs accordingly. Signed-off-by: innocentzero --- .../pacdef_core/src/backend/actual/python.rs | 52 ++++++++++++++----- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/crates/pacdef_core/src/backend/actual/python.rs b/crates/pacdef_core/src/backend/actual/python.rs index 68f0636..454e4a2 100644 --- a/crates/pacdef_core/src/backend/actual/python.rs +++ b/crates/pacdef_core/src/backend/actual/python.rs @@ -38,18 +38,21 @@ impl Backend for Python { fn get_all_installed_packages(&self) -> Result> { let mut cmd = Command::new(self.get_binary()); let output = run_pip_command(&mut cmd, self.get_switches_runtime())?; - - extract_pacdef_packages_pipx(output) + match self.get_binary(){ + "pip" => extract_pacdef_packages(output), + "pipx" => extract_pacdef_packages_pipx(output), + _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), + } } fn get_explicitly_installed_packages(&self) -> Result> { let mut cmd = Command::new(self.get_binary()); - let output = run_pip_command( - &mut cmd, - &["list", "--format", "json", "--not-required", "--user"], - )?; - - extract_pacdef_packages_pipx(output) + let output = run_pip_command(&mut cmd, self.get_switches_explicit())?; + match self.get_binary(){ + "pip" => extract_pacdef_packages(output), + "pipx" => extract_pacdef_packages_pipx(output), + _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), + } } fn make_dependency(&self, _packages: &[Package]) -> Result { @@ -74,7 +77,14 @@ impl Python { fn get_switches_runtime(&self) -> Switches { match self.get_binary() { - "pip" => &["list", "--format", "json", "--user"], + "pip" => &["list", "--format", "json", "--not-required", "--user"], + "pipx" => &["list", "--json"], + _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), + } + } + fn get_switches_explicit(&self) -> Switches { + match self.get_binary() { + "pip" => &["list", "--format", "json", "--user", ""], "pipx" => &["list", "--json"], _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), } @@ -97,11 +107,25 @@ fn extract_pacdef_packages_pipx(value: Value) -> Result> { .as_object() .context("getting inner json object")? .iter() - .map(|(name, _)| { - println!("{name}"); - Package::from(name.as_str()) - }) - // .map(|(name, _)| Package::from(name.as_str())) + .map(|(name, _)| Package::from(name.as_str())) .collect(); Ok(result) } + +// fn extract_pacdef_packages_deps(value: Value) -> Result> { +// let mut alldeps: HashSet = HashSet::new(); +// +// value["venvs"] +// .as_object() +// .context("getting inner json object")? +// .iter() +// .map(|(_, deps_obj)| { +// let deps = deps_obj["metadata"]["main_package"]["apps_paths_of_dependencies"] +// .as_object() +// .iter() +// .map(|(name, _)| Package::from(name)) +// .collect(); +// alldeps.extend(&deps); +// }); +// Ok(alldeps) +// } From b8cfd9e24f09e704cc267ff440a1c21ac6623191 Mon Sep 17 00:00:00 2001 From: innocentzero Date: Sat, 27 Jan 2024 23:35:37 +0530 Subject: [PATCH 4/7] feat(pipx support): Fully supports pipx features Now able to support packages installed via pipx with the exception of packages that do not provide a binary themselves. Install those explicitly via `pipx install --include-deps `. Signed-off-by: innocentzero --- .../pacdef_core/src/backend/actual/python.rs | 54 ++++++++++++------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/crates/pacdef_core/src/backend/actual/python.rs b/crates/pacdef_core/src/backend/actual/python.rs index 454e4a2..4e7f2bf 100644 --- a/crates/pacdef_core/src/backend/actual/python.rs +++ b/crates/pacdef_core/src/backend/actual/python.rs @@ -40,7 +40,18 @@ impl Backend for Python { let output = run_pip_command(&mut cmd, self.get_switches_runtime())?; match self.get_binary(){ "pip" => extract_pacdef_packages(output), - "pipx" => extract_pacdef_packages_pipx(output), + "pipx" => { + let packs = extract_pacdef_packages_pipx(&output); + let deps = extract_pacdef_packages_deps(&output); + match (packs, deps) { + (Ok(mut pack), Ok(deps)) => { + pack.extend(deps.into_iter()); + Ok(pack) + } + (Ok(pack), Err(_)) => Ok(pack), + (Err(pack), _) => Err(pack), + } + } _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), } } @@ -50,7 +61,7 @@ impl Backend for Python { let output = run_pip_command(&mut cmd, self.get_switches_explicit())?; match self.get_binary(){ "pip" => extract_pacdef_packages(output), - "pipx" => extract_pacdef_packages_pipx(output), + "pipx" => extract_pacdef_packages_pipx(&output), _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), } } @@ -102,7 +113,7 @@ fn extract_pacdef_packages(value: Value) -> Result> { Ok(result) } -fn extract_pacdef_packages_pipx(value: Value) -> Result> { +fn extract_pacdef_packages_pipx(value: &Value) -> Result> { let result = value["venvs"] .as_object() .context("getting inner json object")? @@ -112,20 +123,23 @@ fn extract_pacdef_packages_pipx(value: Value) -> Result> { Ok(result) } -// fn extract_pacdef_packages_deps(value: Value) -> Result> { -// let mut alldeps: HashSet = HashSet::new(); -// -// value["venvs"] -// .as_object() -// .context("getting inner json object")? -// .iter() -// .map(|(_, deps_obj)| { -// let deps = deps_obj["metadata"]["main_package"]["apps_paths_of_dependencies"] -// .as_object() -// .iter() -// .map(|(name, _)| Package::from(name)) -// .collect(); -// alldeps.extend(&deps); -// }); -// Ok(alldeps) -// } +fn extract_pacdef_packages_deps(value: &Value) -> Result> { + let mut alldeps: HashSet = HashSet::new(); + let _ = value["venvs"] + .as_object() + .context("getting inner json object")? + .iter() + .map(|(_, deps_obj)| -> Result<()> { + let deps: HashSet = deps_obj["metadata"]["main_package"] + ["apps_paths_of_dependencies"] + .as_object() + .context("getting inner dependencies")? + .iter() + .map(|(name, _)| Package::from(name.as_str())) + .collect(); + + alldeps.extend(deps); + Ok(()) + }); + Ok(alldeps) +} From fb21691e6473803e73fff5f6e83598803ba744f0 Mon Sep 17 00:00:00 2001 From: innocentzero Date: Sat, 27 Jan 2024 23:52:48 +0530 Subject: [PATCH 5/7] fix(remove collection of dependencies) https://github.com/steven-omaha/pacdef/pull/50#issuecomment-1912284202 Signed-off-by: innocentzero --- .../pacdef_core/src/backend/actual/python.rs | 38 ++----------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/crates/pacdef_core/src/backend/actual/python.rs b/crates/pacdef_core/src/backend/actual/python.rs index 4e7f2bf..0427388 100644 --- a/crates/pacdef_core/src/backend/actual/python.rs +++ b/crates/pacdef_core/src/backend/actual/python.rs @@ -40,18 +40,7 @@ impl Backend for Python { let output = run_pip_command(&mut cmd, self.get_switches_runtime())?; match self.get_binary(){ "pip" => extract_pacdef_packages(output), - "pipx" => { - let packs = extract_pacdef_packages_pipx(&output); - let deps = extract_pacdef_packages_deps(&output); - match (packs, deps) { - (Ok(mut pack), Ok(deps)) => { - pack.extend(deps.into_iter()); - Ok(pack) - } - (Ok(pack), Err(_)) => Ok(pack), - (Err(pack), _) => Err(pack), - } - } + "pipx" => extract_pacdef_packages_pipx(output), _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), } } @@ -61,7 +50,7 @@ impl Backend for Python { let output = run_pip_command(&mut cmd, self.get_switches_explicit())?; match self.get_binary(){ "pip" => extract_pacdef_packages(output), - "pipx" => extract_pacdef_packages_pipx(&output), + "pipx" => extract_pacdef_packages_pipx(output), _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), } } @@ -113,7 +102,7 @@ fn extract_pacdef_packages(value: Value) -> Result> { Ok(result) } -fn extract_pacdef_packages_pipx(value: &Value) -> Result> { +fn extract_pacdef_packages_pipx(value: Value) -> Result> { let result = value["venvs"] .as_object() .context("getting inner json object")? @@ -122,24 +111,3 @@ fn extract_pacdef_packages_pipx(value: &Value) -> Result> { .collect(); Ok(result) } - -fn extract_pacdef_packages_deps(value: &Value) -> Result> { - let mut alldeps: HashSet = HashSet::new(); - let _ = value["venvs"] - .as_object() - .context("getting inner json object")? - .iter() - .map(|(_, deps_obj)| -> Result<()> { - let deps: HashSet = deps_obj["metadata"]["main_package"] - ["apps_paths_of_dependencies"] - .as_object() - .context("getting inner dependencies")? - .iter() - .map(|(name, _)| Package::from(name.as_str())) - .collect(); - - alldeps.extend(deps); - Ok(()) - }); - Ok(alldeps) -} From bf01859f05cf8657439ce6b316f90b3c9a42b22f Mon Sep 17 00:00:00 2001 From: innocentzero Date: Sun, 28 Jan 2024 01:01:22 +0530 Subject: [PATCH 6/7] fix(Update README) Added instructions to the README regarding pipx configuration. Signed-off-by: innocentzero --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 57b9c4e..fb5c9f8 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,7 @@ disabled_backends: [] # backends that pacdef should not manage, e.g. ["python"] warn_not_symlinks: true # warn if a group file is not a symlink flatpak_systemwide: true # whether flatpak packages should be installed system-wide or per user +pip_binary: pip # choose whether to use pipx instead of pip for python package management [See [Pitfalls while using pipx](#pitfalls-while-using-pipx)] ``` @@ -219,3 +220,6 @@ Pacdef is supported by [topgrade](https://github.com/topgrade-rs/topgrade). MSRV is 1.70.0 due to dependencies that require this specific version. Development is conducted against the latest stable version. +### Pitfalls while using pipx + +Some packages like [mdformat-myst](https://github.com/executablebooks/mdformat-myst) do not provide an executable themselves but rather act as a plugin to their dependency, which is mdformat in this case. Please install such packages explicitly by running `pipx install --include-deps`. From 34a428f28045f3bf182c13dad3b07f4061c53059 Mon Sep 17 00:00:00 2001 From: innocentzero Date: Sun, 28 Jan 2024 03:11:41 +0530 Subject: [PATCH 7/7] refactor(minor code changes) --- .../pacdef_core/src/backend/actual/python.rs | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/crates/pacdef_core/src/backend/actual/python.rs b/crates/pacdef_core/src/backend/actual/python.rs index 0427388..eecd199 100644 --- a/crates/pacdef_core/src/backend/actual/python.rs +++ b/crates/pacdef_core/src/backend/actual/python.rs @@ -27,6 +27,12 @@ const SWITCHES_REMOVE: Switches = &["uninstall"]; const SUPPORTS_AS_DEPENDENCY: bool = false; +macro_rules! ERROR{ + ($bin:expr) => { + panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx.", $bin) + }; +} + impl Backend for Python { impl_backend_constants!(); @@ -38,21 +44,13 @@ impl Backend for Python { fn get_all_installed_packages(&self) -> Result> { let mut cmd = Command::new(self.get_binary()); let output = run_pip_command(&mut cmd, self.get_switches_runtime())?; - match self.get_binary(){ - "pip" => extract_pacdef_packages(output), - "pipx" => extract_pacdef_packages_pipx(output), - _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), - } + self.extract_packages(output) } fn get_explicitly_installed_packages(&self) -> Result> { let mut cmd = Command::new(self.get_binary()); let output = run_pip_command(&mut cmd, self.get_switches_explicit())?; - match self.get_binary(){ - "pip" => extract_pacdef_packages(output), - "pipx" => extract_pacdef_packages_pipx(output), - _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), - } + self.extract_packages(output) } fn make_dependency(&self, _packages: &[Package]) -> Result { @@ -79,14 +77,22 @@ impl Python { match self.get_binary() { "pip" => &["list", "--format", "json", "--not-required", "--user"], "pipx" => &["list", "--json"], - _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), + _ => ERROR!(self.get_binary()), } } fn get_switches_explicit(&self) -> Switches { match self.get_binary() { - "pip" => &["list", "--format", "json", "--user", ""], + "pip" => &["list", "--format", "json", "--user"], "pipx" => &["list", "--json"], - _ => panic!("Cannot use {} for package management in python. Please use a valid package manager like pip or pipx", self.get_binary()), + _ => ERROR!(self.get_binary()), + } + } + + fn extract_packages(&self, output: Value) -> Result> { + match self.get_binary() { + "pip" => extract_pacdef_packages(output), + "pipx" => extract_pacdef_packages_pipx(output), + _ => ERROR!(self.get_binary()), } } }