SQL Server - ensuring a complex relation on insert
You can use a Foreign Key for this if you add a unique constraint on Application(Id,SecurityPrinicpalType), something like this:
--drop table if exists ApplicationInstallSecurityPrinicpal
--drop table if exists ApplicationInstall
--drop table if exists Application

go

create table Application
(
  Id int identity not null, 
  SecurityPrincipalType int,
  constraint pk_Apppliction
    primary key (Id),
  constraint ak_Application
    unique(Id,SecurityPrincipalType)
)

create table ApplicationInstall
(
  ApplicationId int not null,
  Id int identity not null,
  constraint pk_ApplicationInstall
    primary key (ApplicationId,Id),
  constraint fk_ApplicationInstall_Application
    foreign key (ApplicationID) 
    references Application(Id)
)


create table ApplicationInstallSecurityPrinicpal
(
  ApplicationId int not null,
  ApplicationInstallId int not null,
  Id int identity not null, 
  SecurityPrincipalType int,
  constraint pk_ApplicationInstallSecurityPrinicpal
    primary key (ApplicationId,ApplicationInstallId,Id),
  constraint fk_ApplicationInstallSecurityPrinicpal_ApplictionInstall
    foreign key (ApplicationId,ApplicationInstallId)
    references Application(ApplicationId,Id)
  constraint fk_ApplicationInstallSecurityPrinicpal_Appliction
    foreign key (ApplicationId,SecurityPrincipalType)
    references Application(Id,SecurityPrincipalType)
)