Drop the clang-runtime feature and make this behavior default
The change is imposed by the upstream clang v2 crate change, it's no longer possible to share or send the `Clang` between threads.
MODIFY
Cargo.toml
Cargo.toml
@@ -55,21 +55,21 @@ name = "window"
[dependencies]
libc = "0.2"
num-traits = "0.2"
-once_cell = "1.0"
+once_cell = "1"
# version 0.8.20 doesn't contain the deficiency mentioned in https://deps.rs/crate/opencv/0.59.0#vulnerabilities
rgb = { version = "0.8.20", features = ["argb"], optional = true }
[build-dependencies]
binding-generator = { package = "opencv-binding-generator", version = "0.44.0", path = "binding-generator" }
-cc = { version = "1.0", features = ["parallel"] }
-clang = { version = "1.0", features = ["clang_6_0"] }
-dunce = "1.0"
+cc = { version = "1", features = ["parallel"] }
+clang = { version = "2", features = ["clang_6_0", "runtime"] }
+dunce = "1"
glob = "0.3"
jobserver = "0.1"
-once_cell = "1.0"
+once_cell = "1"
pkg-config = "0.3"
-semver = "1.0"
-shlex = "1.0"
+semver = "1"
+shlex = "1"
# vcpkg-0.2.9 is the first one that has accessible find_vcpkg_root()
vcpkg = "0.2.9"
@@ -77,7 +77,6 @@ vcpkg = "0.2.9"
matches = "0.1"
[features]
-clang-runtime = ["clang/runtime"]
docs-only = []
default = [
"alphamat",

MODIFY
README.md
README.md
@@ -141,13 +141,13 @@ You need to set up the following environment variables to point to the installed
7. You're getting the panic: ```a `libclang` shared library is not loaded on this thread```.
- Enable the `clang-runtime` feature. The reason for the issue is that some crates (like `bindgen`) depend on
- `clang-sys` with hard-enabled `runtime` feature and because of that cargo makes this feature also enabled
- for every other crate that depends on `clang-sys` (`opencv` in this case). During binding generation phase
+ Enable the `clang-runtime` feature or use crate version `0.66` and newer. The reason for the issue is that some crates
+ (like `bindgen`) depend on `clang-sys` with hard-enabled `runtime` feature and because of that cargo makes this feature also
+ enabled for every other crate that depends on `clang-sys` (`opencv` in this case). During binding generation phase
`opencv` crate tries to use multiple threads and `clang-sys` with `runtime` feature enabled doesn't like
that (hence the panic). Enabling `clang-runtime` feature switches to using multiple processes instead of
multiple threads. This makes the build a bit longer because of the need to build the helper binary, but the
- end result is the same.
+ end result is the same. Additionally since crate version `0.66` this behavior is now the default.
## Reporting issues
@@ -263,8 +263,6 @@ The following variables affect the building the of the `opencv` crate, but belon
opencv = { version = ..., default-features = false, features = ["calib3d", "features2d", "flann"]}
```
* `rgb` - allow using [`rgb`](https://crates.io/crates/rgb) crate types as `Mat` elements
-* `clang-runtime` - enables the runtime detection of libclang (`runtime` feature of `clang-sys`). Useful as a
- workaround for when your dependencies (like `bindgen`) pull in `clang-sys` with hard `runtime` feature.
* `docs-only` - internal usage, for building docs on [docs.rs](https://docs.rs/opencv)
## API details

MODIFY
binding-generator/Cargo.toml
binding-generator/Cargo.toml
@@ -10,10 +10,10 @@ edition = "2018"
[lib]
[dependencies]
-clang = {version = "1.0", features = ["clang_6_0"]}
+clang = {version = "2", features = ["clang_6_0"]}
clang-sys = {version = "1.0", features = ["clang_6_0"]}
-dunce = "1.0"
-maplit = "1.0"
-once_cell = "1.0"
-percent-encoding = "2.0"
-regex = "1.0"
+dunce = "1"
+maplit = "1"
+once_cell = "1"
+percent-encoding = "2"
+regex = "1"

MODIFY
build.rs
build.rs
@@ -4,8 +4,9 @@ use std::{
ffi::OsStr,
fs::File,
io::{BufRead, BufReader},
+ io,
path::{Path, PathBuf},
- process::Command,
+ process::{Child, Command, Stdio},
};
use glob::glob;
@@ -219,6 +220,34 @@ fn build_compiler(opencv: &Library) -> cc::Build {
out
}
+fn build_job_server() -> Option<jobserver::Client> {
+ unsafe { jobserver::Client::from_env() }
+ .or_else(|| {
+ let num_jobs = env::var("NUM_JOBS").ok()
+ .and_then(|jobs| jobs.parse().ok())
+ .unwrap_or(2)
+ .max(1);
+ eprintln!("=== Creating a new job server with num_jobs: {}", num_jobs);
+ jobserver::Client::new(num_jobs).ok()
+ })
+}
+
+// todo: replace by https://github.com/rust-lang/cargo/issues/9096 when stable
+fn build_clang_generator() -> io::Result<Child> {
+ let cargo_bin = PathBuf::from(env::var_os("CARGO").unwrap_or_else(|| "cargo".into()));
+ let mut cargo = Command::new(cargo_bin);
+ // generator script is quite slow in debug mode, so we force it to be built in release mode
+ cargo.args(&["build", "--release", "--package", "opencv-binding-generator", "--bin", "binding-generator"])
+ .env("CARGO_TARGET_DIR", &*OUT_DIR);
+ if let Some(host_triple) = HOST_TRIPLE.as_ref() {
+ cargo.args(&["--target", host_triple]);
+ }
+ println!("running: {:?}", &cargo);
+ cargo.stdout(Stdio::piped());
+ cargo.stderr(Stdio::piped());
+ cargo.spawn()
+}
+
fn setup_rerun() -> Result<()> {
for &v in ENV_VARS.iter() {
println!("cargo:rerun-if-env-changed={}", v);
@@ -271,20 +300,8 @@ fn main() -> Result<()> {
return Ok(());
}
- let generator_build = if cfg!(feature = "clang-runtime") { // start building binding generator as early as possible
- let cargo_bin = PathBuf::from(env::var_os("CARGO").unwrap_or_else(|| "cargo".into()));
- let mut cargo = Command::new(cargo_bin);
- // generator script is quite slow in debug mode, so we force it to be built in release mode
- cargo.args(&["build", "--release", "--package", "opencv-binding-generator", "--bin", "binding-generator"])
- .env("CARGO_TARGET_DIR", &*OUT_DIR);
- if let Some(host_triple) = HOST_TRIPLE.as_ref() {
- cargo.args(&["--target", host_triple]);
- }
- println!("running: {:?}", &cargo);
- Some(cargo.spawn()?)
- } else {
- None
- };
+ let job_server = build_job_server().ok_or("Can't create job server")?;
+ let generator_build = build_clang_generator()?;
eprintln!("=== Crate version: {:?}", env::var_os("CARGO_PKG_VERSION"));
eprintln!("=== Environment configuration:");
@@ -345,7 +362,7 @@ fn main() -> Result<()> {
setup_rerun()?;
- generator::gen_wrapper(opencv_header_dir, &opencv, generator_build)?;
+ generator::gen_wrapper(opencv_header_dir, &opencv, job_server, generator_build)?;
build_wrapper(&opencv);
// -l linker args should be emitted after -l static
opencv.emit_cargo_metadata();

MODIFY
build/generator.rs
build/generator.rs
use std::{
- env,
ffi::OsStr,
fs::{self, DirEntry, File, OpenOptions},
io::{self, BufRead, BufReader, Write},
@@ -51,7 +50,7 @@ fn file_move_to_dir(src_file: &Path, target_dir: &Path) -> Result<PathBuf> {
Ok(target_file)
}
-pub fn gen_wrapper(opencv_header_dir: &Path, opencv: &Library, generator_build: Option<Child>) -> Result<()> {
+pub fn gen_wrapper(opencv_header_dir: &Path, opencv: &Library, job_server: jobserver::Client, generator_build: Child) -> Result<()> {
let out_dir_as_str = OUT_DIR.to_str().unwrap();
let target_hub_dir = SRC_DIR.join("opencv");
let target_module_dir = target_hub_dir.join("hub");
@@ -61,8 +60,6 @@ pub fn gen_wrapper(opencv_header_dir: &Path, opencv: &Library, generator_build:
eprintln!("=== Placing generated bindings into: {}", target_hub_dir.display());
eprintln!("=== Using OpenCV headers from: {}", opencv_header_dir.display());
- let modules = MODULES.get().expect("MODULES not initialized");
-
for entry in read_dir(&OUT_DIR)? {
let path = entry.path();
if path.is_file() && path.extension().and_then(OsStr::to_str).map_or(true, |ext| !ext.eq_ignore_ascii_case("dll")) {
@@ -74,27 +71,32 @@ pub fn gen_wrapper(opencv_header_dir: &Path, opencv: &Library, generator_build:
.filter(|&include_path| include_path != opencv_header_dir)
.cloned()
.collect::<Vec<_>>();
- let num_jobs = env::var("NUM_JOBS").ok()
- .and_then(|jobs| jobs.parse().ok())
- .unwrap_or(2);
- let job_server = jobserver::Client::new(num_jobs).expect("Can't create job server");
- let mut join_handles = Vec::with_capacity(modules.len());
- let start;
+
let clang = clang::Clang::new().expect("Cannot initialize clang");
- println!("=== Clang: {}", clang::get_version());
+ eprintln!("=== Clang: {}", clang::get_version());
let gen = binding_generator::Generator::new(opencv_header_dir, &additional_include_dirs, &*SRC_CPP_DIR, clang);
+ eprintln!("=== Clang command line args: {:#?}", gen.build_clang_command_line_args());
+
+ eprintln!("=== Waiting until the binding-generator binary is built...");
+ let res = generator_build.wait_with_output()?;
+ if let Err(e) = io::stdout().write(&res.stdout) {
+ eprintln!("=== Can't write stdout: {:?}, error: {}", res.stdout, e)
+ }
+ if let Err(e) = io::stderr().write(&res.stderr) {
+ eprintln!("=== Can't write stderr: {:?}, error: {}", res.stdout, e)
+ }
+ if !res.status.success() {
+ return Err("Failed to build the bindings generator".into());
+ }
+
let additional_include_dirs = Arc::new(additional_include_dirs.iter().cloned()
.map(|p| p.to_str().expect("Can't convert additional include dir to UTF-8 string").to_string())
.collect::<Vec<_>>()
);
- eprintln!("=== Clang command line args: {:#?}", gen.build_clang_command_line_args());
- if cfg!(feature = "clang-runtime") {
- let status = generator_build.expect("Impossible").wait()?;
- if !status.success() {
- return Err("Failed to build the bindings generator".into());
- }
let opencv_header_dir = Arc::new(opencv_header_dir.to_owned());
- start = Instant::now();
+ let modules = MODULES.get().expect("MODULES not initialized");
+ let mut join_handles = Vec::with_capacity(modules.len());
+ let start = Instant::now();
modules.iter().for_each(|module| {
let token = job_server.acquire().expect("Can't acquire token from job server");
let join_handle = thread::spawn({
@@ -110,7 +112,7 @@ pub fn gen_wrapper(opencv_header_dir: &Path, opencv: &Library, generator_build:
.arg(&*OUT_DIR)
.arg(&module)
.arg(additional_include_dirs.join(","));
- eprintln!("=== Running binding generator binary: {:#?}", bin_generator);
+ eprintln!("=== Running: {:#?}", bin_generator);
let res = bin_generator.status().expect("Can't run bindings generator");
if !res.success() {
panic!("Failed to run the bindings generator");
@@ -121,30 +123,6 @@ pub fn gen_wrapper(opencv_header_dir: &Path, opencv: &Library, generator_build:
});
join_handles.push(join_handle);
});
- } else {
- let gen = Arc::new(gen);
- start = Instant::now();
- modules.iter().for_each(|module| {
- let token = job_server.acquire().expect("Can't acquire token from job server");
- let opencv_version = opencv.version.to_string();
- let join_handle = thread::spawn({
- let gen = Arc::clone(&gen);
- move || {
- let bindings_writer = binding_generator::writer::RustNativeBindingWriter::new(
- &*SRC_CPP_DIR,
- &*OUT_DIR,
- module,
- &opencv_version,
- false,
- );
- gen.process_opencv_module(module, bindings_writer);
- eprintln!("=== Generated: {}", module);
- drop(token); // needed to move the token to the thread
- }
- });
- join_handles.push(join_handle);
- });
- }
for join_handle in join_handles {
join_handle.join().expect("Generator thread panicked");
}

MODIFY
ci/script.sh
ci/script.sh
@@ -55,9 +55,8 @@ FEATURES=rgb
cargo test -vv --features "$FEATURES"
cargo test --release -vv --features "$FEATURES"
-cargo test --release -vv --features "$FEATURES,clang-runtime"
-pushd ci/test-proj-clang-runtime
+pushd ci/test-proj
cargo run -vv
popd

RENAME
ci/test-proj-clang-runtime/.gitignore
ci/test-proj/.gitignore
RENAME
ci/test-proj-clang-runtime/Cargo.toml
ci/test-proj/Cargo.toml
[package]
-name = "test-proj-clang-runtime"
+name = "test-proj"
description = "Helper to test whether project using opencv dependency compiles"
version = "0.1.0"
authors = ["Pro"]
edition = "2018"
[dependencies]
-opencv = { path = "../..", features = ["clang-runtime"] }
+opencv = { path = "../.." }
[workspace]

RENAME
ci/test-proj-clang-runtime/src/main.rs
ci/test-proj/src/main.rs
Drop the clang-runtime feature and make this behavior default

The change is imposed by the upstream clang v2 crate change, it's no longer possible to share or send the `Clang` between threads.