title: CMake cant find the external libraries when subdirectory is used to build dependencies for each target
body: <p>I have a set of libraries and their respective tests, and they are organized in the following fashion:</p>
  <pre><code>-Lib1
  -Lib1_Test
  -Lib2
  -Lib2_Test
  -Lib3
  -Lib3_Test
  ....
  </code></pre>
  <p>and so on. some of these libs depend on others, for example <code>Lib1</code> depends on <code>Lib2</code>, and Lib3 depends on Lib1. I can easily build each target separately and all is fine. However, When I try to build them in such a way that each target first builds its dependencies and then itself I face an issue here.<br />
  For this very purpose, I used 'add_subdirectory` for each dependenant target and I also check if the target is already built or not so only if its not, we include it and build it.<br />
  This is how one of my CMakeLists.txt look like :</p>
  <pre><code>cmake_minimum_required(VERSION 3.11)
  
  project(FV)
  
  set(CMAKE_CXX_STANDARD 17)
  set( CMAKE_CXX_FLAGS  &quot;${CMAKE_CXX_FLAGS} -fPIC&quot; )
  
  find_package(Torch REQUIRED)
  find_package(OpenCV REQUIRED)
  find_package(Protobuf REQUIRED)
  
  if (&quot;${TORCH_LIB_DIRS}&quot; STREQUAL &quot;&quot;)
    set(TORCH_LIB_DIRS /home/rika/libtorch-cxx11-abi-shared-with-deps-1.6.0+cpu/libtorch/lib)
  endif()
  
  get_filename_component(PARENT_DIR ${PROJECT_SOURCE_DIR} DIRECTORY)
  
  #if (DLL)
  add_definitions(-D_FV_BUILD_DLL)
  set(CMAKE_INSTALL_PREFIX ${PARENT_DIR}/built_stuff)
  
  if(NOT TARGET AntiSpoofer)
    add_subdirectory(${PARENT_DIR}/AntiSpoofer ${PARENT_DIR}/AntiSpoofer/build)
  endif()
  
  if(NOT TARGET Detector)
    add_subdirectory(${PARENT_DIR}/Detector ${PARENT_DIR}/Detector/build)
  endif()
  
  if(NOT TARGET Hashing)
    add_subdirectory(${PARENT_DIR}/Hashing ${PARENT_DIR}/Hashing/build)
  endif()
  
  include_directories(${PARENT_DIR}/Dependencies/include ${Protobuf_INCLUDE_DIRS} ${TORCH_INCLUDE_DIRS} ${OpenCV_INCLUDE_DIRS})
  
  set(FV_H ${PARENT_DIR}/Dependencies/include/FV/FV.h)
  set(Config_H ${PARENT_DIR}/Dependencies/include/messages/Config.pb.h)
  set(TorchSerializer_H ${PARENT_DIR}/Dependencies/include/messages/TorchSerializer.pb.h)
  set(TorchSerializer_PY ${PARENT_DIR}/Dependencies/include/messages/TorchSerializer_pb2.py)
  
  set(FV_SRC  ./FV.cpp)
  set(Config_SRC ${PARENT_DIR}/Dependencies/include/messages/Config.pb.cc)
  set(TorchSerializer_SRC ${PARENT_DIR}/Dependencies/include/messages/TorchSerializer.pb.cc)
  
  LINK_DIRECTORIES(${TORCH_LIB_DIRS})
  LINK_DIRECTORIES(${PARENT_DIR}/AntiSpoofer/build)
  LINK_DIRECTORIES(${PARENT_DIR}/Blinker/build)
  LINK_DIRECTORIES(${PARENT_DIR}/Detector/build)
  LINK_DIRECTORIES(${PARENT_DIR}/Hashing/build)
  
  add_library(
      FV
      SHARED
     ${FV_SRC}
     ${Config_SRC}
     ${TorchSerializer_SRC}
    )
  
  target_link_directories(FV PUBLIC ${PARENT_DIR}/built_stuff/lib)
  
  # Link
  target_link_libraries(FV ${OpenCV_LIBS})
  target_link_libraries(FV ${TORCH_LIB_DIRS}/libc10.so)
  target_link_libraries(FV ${TORCH_LIB_DIRS}/libtorch_cpu.so)
  
  target_link_libraries(FV Hashing)
  target_link_libraries(FV Blinker)
  target_link_libraries(FV AntiSpoofer)
  target_link_libraries(FV Detector)
  
  # add FV include
  install(TARGETS FV LIBRARY DESTINATION lib)
  install(FILES ${FV_H} DESTINATION include/FV)
  
  # add protobuf related headers/srcs for c++ and python
  install(FILES ${Config_H} DESTINATION include/messages)
  install(FILES ${TorchSerializer_H} DESTINATION include/messages)
  install(FILES ${Config_SRC} DESTINATION src/messages)
  install(FILES ${TorchSerializer_SRC} DESTINATION src/messages)
  install(FILES ${TorchSerializer_PY} DESTINATION Python)
  </code></pre>
  <p>when I try to call this very cmakelists like this :</p>
  <pre><code>cmake -DCMAKE_PREFIX_PATH=&quot;/home/rika/libtorch-cxx11-abi-shared-with-deps-1.6.0+cpu/libtorch/share/cmake&quot; ..
  </code></pre>
  <p>everything builds just fine and all targets seem to be able to get and use the passed <code>CMAKE_PREFIX_PATH</code> that I send here. However, just after this, if I <code>cd</code> into the <code>FV_Test</code> which by the way is given below, it fails with the error messages indicating the failure of CMake in finding torch! which is clearly specified in the argument when calling CMake. This is the error message I get after trying to build FV_Test:</p>
  <pre><code>-- The C compiler identification is GNU 9.3.0
  -- The CXX compiler identification is GNU 9.3.0
  -- Detecting C compiler ABI info
  -- Detecting C compiler ABI info - done
  -- Check for working C compiler: /usr/bin/cc - skipped
  -- Detecting C compile features
  -- Detecting C compile features - done
  -- Detecting CXX compiler ABI info
  -- Detecting CXX compiler ABI info - done
  -- Check for working CXX compiler: /usr/bin/c++ - skipped
  -- Detecting CXX compile features
  -- Detecting CXX compile features - done
  -- Found OpenCV: /usr/local (found version &quot;3.4.10&quot;) 
  -- Found Protobuf: /usr/local/lib/libprotobuf.so;-lpthread (found version &quot;3.13.0&quot;) 
  -- Found Threads: TRUE  
  -- Found Torch: /home/rika/libtorch-cxx11-abi-shared-with-deps-1.6.0+cpu/libtorch/lib/libtorch.so  
  -- Using CMake version: 3.18.2
  -- Compiling dlib version: 19.21.0
  -- Found X11: /usr/include   
  -- Looking for XOpenDisplay in /usr/lib/x86_64-linux-gnu/libX11.so;/usr/lib/x86_64-linux-gnu/libXext.so
  -- Looking for XOpenDisplay in /usr/lib/x86_64-linux-gnu/libX11.so;/usr/lib/x86_64-linux-gnu/libXext.so - found
  -- Looking for gethostbyname
  -- Looking for gethostbyname - found
  -- Looking for connect
  -- Looking for connect - found
  -- Looking for remove
  -- Looking for remove - found
  -- Looking for shmat
  -- Looking for shmat - found
  -- Looking for IceConnectionNumber in ICE
  -- Looking for IceConnectionNumber in ICE - found
  -- Found system copy of libpng: /usr/lib/x86_64-linux-gnu/libpng.so;/usr/lib/x86_64-linux-gnu/libz.so
  -- Found system copy of libjpeg: /usr/lib/x86_64-linux-gnu/libjpeg.so
  -- Searching for BLAS and LAPACK
  -- Searching for BLAS and LAPACK
  -- Found PkgConfig: /usr/bin/pkg-config (found version &quot;0.29.1&quot;) 
  -- Checking for module 'cblas'
  --   No package 'cblas' found
  -- Checking for module 'lapack'
  --   Found lapack, version 3.10.3
  -- Looking for sys/types.h
  -- Looking for sys/types.h - found
  -- Looking for stdint.h
  -- Looking for stdint.h - found
  -- Looking for stddef.h
  -- Looking for stddef.h - found
  -- Check size of void*
  -- Check size of void* - done
  -- Found Intel MKL BLAS/LAPACK library
  -- Looking for sgesv
  -- Looking for sgesv - found
  -- Looking for sgesv_
  -- Looking for sgesv_ - found
  CUDA_TOOLKIT_ROOT_DIR not found or specified
  -- Could NOT find CUDA (missing: CUDA_TOOLKIT_ROOT_DIR CUDA_NVCC_EXECUTABLE CUDA_INCLUDE_DIRS CUDA_CUDART_LIBRARY) (Required is at least version &quot;7.5&quot;)
  -- DID NOT FIND CUDA
  -- Disabling CUDA support for dlib.  DLIB WILL NOT USE CUDA
  -- C++11 activated.
  -- Configuring done
  CMake Error at CMakeLists.txt:58 (target_link_libraries):
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch_cpu,INTERFACE_COMPILE_DEFINITIONS&gt;
  
    Target &quot;torch_cpu&quot; not found.
  
  
  CMake Error at CMakeLists.txt:58 (target_link_libraries):
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch,INTERFACE_COMPILE_DEFINITIONS&gt;
  
    Target &quot;torch&quot; not found.
  
  
  CMake Error at CMakeLists.txt:58 (target_link_libraries):
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch_cpu,INTERFACE_INCLUDE_DIRECTORIES&gt;
  
    Target &quot;torch_cpu&quot; not found.
  
  
  CMake Error at CMakeLists.txt:58 (target_link_libraries):
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch,INTERFACE_INCLUDE_DIRECTORIES&gt;
  
    Target &quot;torch&quot; not found.
  
  
  CMake Error:
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch_cpu,INTERFACE_SYSTEM_INCLUDE_DIRECTORIES&gt;
  
    Target &quot;torch_cpu&quot; not found.
  
  
  CMake Error:
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch_cpu,INTERFACE_INCLUDE_DIRECTORIES&gt;
  
    Target &quot;torch_cpu&quot; not found.
  
  
  CMake Error:
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch,INTERFACE_SYSTEM_INCLUDE_DIRECTORIES&gt;
  
    Target &quot;torch&quot; not found.
  
  
  CMake Error:
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch,INTERFACE_INCLUDE_DIRECTORIES&gt;
  
    Target &quot;torch&quot; not found.
  
  
  CMake Error at CMakeLists.txt:58 (target_link_libraries):
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch_cpu,INTERFACE_COMPILE_OPTIONS&gt;
  
    Target &quot;torch_cpu&quot; not found.
  
  
  CMake Error at CMakeLists.txt:58 (target_link_libraries):
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch,INTERFACE_COMPILE_OPTIONS&gt;
  
    Target &quot;torch&quot; not found.
  
  
  CMake Warning at CMakeLists.txt:49 (add_executable):
    Cannot generate a safe runtime search path for target FV_test because files
    in some directories may conflict with libraries in implicit directories:
  
      runtime library [libpng16.so.16] in /usr/lib/x86_64-linux-gnu may be hidden by files in:
        /home/rika/anaconda3/lib
      runtime library [libz.so.1] in /usr/lib/x86_64-linux-gnu may be hidden by files in:
        /home/rika/anaconda3/lib
  
    Some of these libraries may not be found correctly.
  
  
  CMake Error at CMakeLists.txt:58 (target_link_libraries):
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch_cpu,INTERFACE_COMPILE_DEFINITIONS&gt;
  
    Target &quot;torch_cpu&quot; not found.
  
  
  CMake Error at CMakeLists.txt:58 (target_link_libraries):
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch,INTERFACE_COMPILE_DEFINITIONS&gt;
  
    Target &quot;torch&quot; not found.
  
  
  CMake Error at CMakeLists.txt:58 (target_link_libraries):
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch_cpu,INTERFACE_INCLUDE_DIRECTORIES&gt;
  
    Target &quot;torch_cpu&quot; not found.
  
  
  CMake Error at CMakeLists.txt:58 (target_link_libraries):
    Error evaluating generator expression:
  
      $&lt;TARGET_PROPERTY:torch,INTERFACE_INCLUDE_DIRECTORIES&gt;
  
    Target &quot;torch&quot; not found.
  
  
  CMake Warning at /home/rika/Desktop/LibtorchPort/FV/CMakeLists.txt:68 (add_library):
    Cannot generate a safe runtime search path for target FV because files in
    some directories may conflict with libraries in implicit directories:
  
      runtime library [libpng16.so.16] in /usr/lib/x86_64-linux-gnu may be hidden by files in:
        /home/rika/anaconda3/lib
      runtime library [libz.so.1] in /usr/lib/x86_64-linux-gnu may be hidden by files in:
        /home/rika/anaconda3/lib
  
    Some of these libraries may not be found correctly.
  
  
  CMake Warning at /home/rika/Desktop/LibtorchPort/AntiSpoofer/CMakeLists.txt:57 (add_library):
    Cannot generate a safe runtime search path for target AntiSpoofer because
    files in some directories may conflict with libraries in implicit
    directories:
  
      runtime library [libpng16.so.16] in /usr/lib/x86_64-linux-gnu may be hidden by files in:
        /home/rika/anaconda3/lib
      runtime library [libz.so.1] in /usr/lib/x86_64-linux-gnu may be hidden by files in:
        /home/rika/anaconda3/lib
  
    Some of these libraries may not be found correctly.
  
  
  CMake Warning at /home/rika/Desktop/LibtorchPort/Blinker/CMakeLists.txt:46 (add_library):
    Cannot generate a safe runtime search path for target Blinker because files
    in some directories may conflict with libraries in implicit directories:
  
      runtime library [libpng16.so.16] in /usr/lib/x86_64-linux-gnu may be hidden by files in:
        /home/rika/anaconda3/lib
      runtime library [libz.so.1] in /usr/lib/x86_64-linux-gnu may be hidden by files in:
        /home/rika/anaconda3/lib
  
    Some of these libraries may not be found correctly.
  
  
  -- Generating done
  CMake Generate step failed.  Build files cannot be regenerated correctly.
  </code></pre>
  <p>And this is the CMakeLists.txt that I'm using for <code>FV_Test</code>.</p>
  <pre><code>cmake_minimum_required(VERSION 3.11)
  project(FV_Test)
  
  set(CMAKE_CXX_STANDARD 17)
  
  find_package(OpenCV REQUIRED)
  find_package(Protobuf REQUIRED)
  
  if (&quot;${TORCH_LIB_DIRS}&quot; STREQUAL &quot;&quot;)
    set(TORCH_LIB_DIRS /home/rika/libtorch-cxx11-abi-shared-with-deps-1.6.0+cpu/libtorch/lib)
  endif()
  
  get_filename_component(PARENT_DIR ${PROJECT_SOURCE_DIR} DIRECTORY)
  
  if(NOT TARGET FV)
    add_subdirectory(${PARENT_DIR}/FV ${PARENT_DIR}/FV/build)
  endif()
  
  set(CMAKE_INSTALL_PREFIX ${PARENT_DIR}/built_stuff)
  
  include_directories(${PARENT_DIR}/Dependencies/include ${OpenCV_INCLUDE_DIRS} ${Protobuf_INCLUDE_DIRS}) # 
  
  LINK_DIRECTORIES(${TORCH_LIB_DIRS})
  LINK_DIRECTORIES(${PARENT_DIR}/AntiSpoofer/build)
  LINK_DIRECTORIES(${PARENT_DIR}/Blinker/build)
  LINK_DIRECTORIES(${PARENT_DIR}/FV/build)
  
  add_executable(FV_test ./FV_Test.cpp )
  target_link_directories(FV_test PUBLIC ${PARENT_DIR}/AntiSpoofer/build)
  target_link_directories(FV_test PUBLIC ${PARENT_DIR}/Blinker/build)
  target_link_directories(FV_test PUBLIC ${PARENT_DIR}/FV/build)
  
  # Link
  target_link_libraries(FV_test ${OpenCV_LIBS})
  target_link_libraries(FV_test ${Protobuf_LIBRARIES})
  target_link_libraries(FV_test FV)
  target_link_libraries(FV_test Blinker)
  
  install(TARGETS FV_test DESTINATION bin)
  </code></pre>
  <p>Why am I seeing this behavior and how can I solve this?</p>